Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle bad data gracefully.
kishorenc11c41804e5ca
diff --git a/include/collection.h b/include/collection.hindex 184925ed..738db45d 100644--- a/include/collection.h+++ b/include/collection.h@@ -323,7 +323,7 @@ private:const std::vector<Index*> indices;- const bool index_all_fields;+ const std::atomic<bool> index_all_fields;// methods@@ -349,6 +349,8 @@ private:std::map<size_t, std::vector<uint32_t>>& include_ids,std::vector<uint32_t> & excluded_ids) const;+ Option<bool> check_and_update_schema(nlohmann::json& document);+static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,const std::pair<uint64_t, facet_count_t>& b) {return std::tie(a.second.count, a.first) > std::tie(b.second.count, b.first);@@ -457,7 +459,8 @@ public:Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);+ Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,+ bool is_update, const DIRTY_VALUES& dirty_values);static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,const spp::sparse_hash_set<std::string> & exclude_fields);@@ -490,10 +493,12 @@ public:size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);Option<nlohmann::json> add(const std::string & json_str,- const index_operation_t& operation=CREATE, const std::string& id="");+ const index_operation_t& operation=CREATE, const std::string& id="",+ const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT);nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,- const index_operation_t& operation=CREATE, const std::string& id="");+ const index_operation_t& operation=CREATE, const std::string& id="",+ const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT);Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,@@ -534,6 +539,8 @@ public:size_t get_num_documents() const;+ DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;+// Override operationsOption<uint32_t> add_override(const override_t & override);diff --git a/include/field.h b/include/field.hindex cf48da5a..121f5e51 100644--- a/include/field.h+++ b/include/field.h@@ -105,6 +105,10 @@ struct field {type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);}+ bool is_sortable() const {+ return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();+ }+bool has_valid_type() const {return is_string() || is_integer() || is_float() || is_bool() || is_geopoint();}diff --git a/include/index.h b/include/index.hindex 36811d9b..c2076b36 100644--- a/include/index.h+++ b/include/index.h@@ -17,6 +17,7 @@#include <set>#include "string_utils.h"#include "num_tree.h"+#include "magic_enum.hpp"struct token_t {size_t position;@@ -103,6 +104,12 @@ enum index_operation_t {DELETE};+enum class DIRTY_VALUES {+ COERCE_OR_IGNORE = 1,+ COERCE_OR_REJECT = 2,+ REJECT = 3,+ IGNORE = 4,+};struct index_record {size_t position; // position of record in the original request@@ -118,8 +125,12 @@ struct index_record {Option<bool> indexed; // indicates if the indexing operation was a success- index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation):- position(record_pos), seq_id(seq_id), doc(doc), operation(operation), is_update(false), indexed(false) {+ const DIRTY_VALUES dirty_values;++ index_record(size_t record_pos, uint32_t seq_id, const nlohmann::json& doc, index_operation_t operation,+ const DIRTY_VALUES& dirty_values):+ position(record_pos), seq_id(seq_id), doc(doc), operation(operation), is_update(false),+ indexed(false), dirty_values(dirty_values) {}@@ -227,6 +238,21 @@ private:static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);+ static Option<uint32_t> coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index);++ static Option<uint32_t> coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index);++ static Option<uint32_t> coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index);++ static Option<uint32_t> coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index);++ static Option<uint32_t> coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index);+public:// for limiting number of results on multiple candidates / query rewritesenum {TYPO_TOKENS_THRESHOLD = 100};@@ -335,7 +361,8 @@ public:std::vector<index_record> & iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema);+ const std::map<std::string, field> & facet_schema,+ bool index_all_fields);static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,const std::vector<uint32_t*>& leaf_to_indices,@@ -348,13 +375,15 @@ public:uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;- static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,+ static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,const std::map<std::string, field> & facet_schema,- bool is_update);+ bool is_update,+ bool index_all_fields,+ const DIRTY_VALUES& dirty_values);- void refresh_search_schema(const std::unordered_map<std::string, field>& src_search_schema);+ void refresh_schemas(const std::vector<field>& new_fields);};diff --git a/include/magic_enum.hpp b/include/magic_enum.hppnew file mode 100644index 00000000..aa17bc6e--- /dev/null+++ b/include/magic_enum.hpp@@ -0,0 +1,1131 @@+// __ __ _ ______ _____+// | \/ | (_) | ____| / ____|_ _+// | \ / | __ _ __ _ _ ___ | |__ _ __ _ _ _ __ ___ | | _| |_ _| |_+// | |\/| |/ _` |/ _` | |/ __| | __| | '_ \| | | | '_ ` _ \ | | |_ _|_ _|+// | | | | (_| | (_| | | (__ | |____| | | | |_| | | | | | | | |____|_| |_|+// |_| |_|\__,_|\__, |_|\___| |______|_| |_|\__,_|_| |_| |_| \_____|+// __/ | https://github.com/Neargye/magic_enum+// |___/ version 0.7.2+//+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.+// SPDX-License-Identifier: MIT+// Copyright (c) 2019 - 2021 Daniil Goncharov <[email protected]>.+//+// Permission is hereby granted, free of charge, to any person obtaining a copy+// of this software and associated documentation files (the "Software"), to deal+// in the Software without restriction, including without limitation the rights+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+// copies of the Software, and to permit persons to whom the Software is+// furnished to do so, subject to the following conditions:+//+// The above copyright notice and this permission notice shall be included in all+// copies or substantial portions of the Software.+//+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+// SOFTWARE.++#ifndef NEARGYE_MAGIC_ENUM_HPP+#define NEARGYE_MAGIC_ENUM_HPP++#define MAGIC_ENUM_VERSION_MAJOR 0+#define MAGIC_ENUM_VERSION_MINOR 7+#define MAGIC_ENUM_VERSION_PATCH 2++#include <array>+#include <cassert>+#include <cstdint>+#include <cstddef>+#include <iosfwd>+#include <limits>+#include <type_traits>+#include <utility>++#if !defined(MAGIC_ENUM_USING_ALIAS_OPTIONAL)+#include <optional>+#endif+#if !defined(MAGIC_ENUM_USING_ALIAS_STRING)+#include <string>+#endif+#if !defined(MAGIC_ENUM_USING_ALIAS_STRING_VIEW)+#include <string_view>+#endif++#if defined(__clang__)+# pragma clang diagnostic push+#elif defined(__GNUC__)+# pragma GCC diagnostic push+# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" // May be used uninitialized 'return {};'.+#elif defined(_MSC_VER)+# pragma warning(push)+# pragma warning(disable : 26495) // Variable 'static_string<N>::chars_' is uninitialized.+# pragma warning(disable : 28020) // Arithmetic overflow: Using operator '-' on a 4 byte value and then casting the result to a 8 byte value.+# pragma warning(disable : 26451) // The expression '0<=_Param_(1)&&_Param_(1)<=1-1' is not true at this call.+#endif++// Checks magic_enum compiler compatibility.+#if defined(__clang__) && __clang_major__ >= 5 || defined(__GNUC__) && __GNUC__ >= 9 || defined(_MSC_VER) && _MSC_VER >= 1910+# undef MAGIC_ENUM_SUPPORTED+# define MAGIC_ENUM_SUPPORTED 1+#endif++// Checks magic_enum compiler aliases compatibility.+#if defined(__clang__) && __clang_major__ >= 5 || defined(__GNUC__) && __GNUC__ >= 9 || defined(_MSC_VER) && _MSC_VER >= 1920+# undef MAGIC_ENUM_SUPPORTED_ALIASES+# define MAGIC_ENUM_SUPPORTED_ALIASES 1+#endif++// Enum value must be greater or equals than MAGIC_ENUM_RANGE_MIN. By default MAGIC_ENUM_RANGE_MIN = -128.+// If need another min range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MIN.+#if !defined(MAGIC_ENUM_RANGE_MIN)+# define MAGIC_ENUM_RANGE_MIN -128+#endif++// Enum value must be less or equals than MAGIC_ENUM_RANGE_MAX. By default MAGIC_ENUM_RANGE_MAX = 128.+// If need another max range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MAX.+#if !defined(MAGIC_ENUM_RANGE_MAX)+# define MAGIC_ENUM_RANGE_MAX 128+#endif++namespace magic_enum {++// If need another optional type, define the macro MAGIC_ENUM_USING_ALIAS_OPTIONAL.+#if defined(MAGIC_ENUM_USING_ALIAS_OPTIONAL)+ MAGIC_ENUM_USING_ALIAS_OPTIONAL+#else+ template <typename T>+ using optional = std::optional<T>;+#endif++// If need another string_view type, define the macro MAGIC_ENUM_USING_ALIAS_STRING_VIEW.+#if defined(MAGIC_ENUM_USING_ALIAS_STRING_VIEW)+ MAGIC_ENUM_USING_ALIAS_STRING_VIEW+#else+ using string_view = std::string_view;+#endif++// If need another string type, define the macro MAGIC_ENUM_USING_ALIAS_STRING.+#if defined(MAGIC_ENUM_USING_ALIAS_STRING)+ MAGIC_ENUM_USING_ALIAS_STRING+#else+ using string = std::string;+#endif++ namespace customize {++// Enum value must be in range [MAGIC_ENUM_RANGE_MIN, MAGIC_ENUM_RANGE_MAX]. By default MAGIC_ENUM_RANGE_MIN = -128, MAGIC_ENUM_RANGE_MAX = 128.+// If need another range for all enum types by default, redefine the macro MAGIC_ENUM_RANGE_MIN and MAGIC_ENUM_RANGE_MAX.+// If need another range for specific enum type, add specialization enum_range for necessary enum type.+ template <typename E>+ struct enum_range {+ static_assert(std::is_enum_v<E>, "magic_enum::customize::enum_range requires enum type.");+ inline static constexpr int min = MAGIC_ENUM_RANGE_MIN;+ inline static constexpr int max = MAGIC_ENUM_RANGE_MAX;+ static_assert(max > min, "magic_enum::customize::enum_range requires max > min.");+ };++ static_assert(MAGIC_ENUM_RANGE_MIN <= 0, "MAGIC_ENUM_RANGE_MIN must be less or equals than 0.");+ static_assert(MAGIC_ENUM_RANGE_MIN > (std::numeric_limits<std::int16_t>::min)(), "MAGIC_ENUM_RANGE_MIN must be greater than INT16_MIN.");++ static_assert(MAGIC_ENUM_RANGE_MAX > 0, "MAGIC_ENUM_RANGE_MAX must be greater than 0.");+ static_assert(MAGIC_ENUM_RANGE_MAX < (std::numeric_limits<std::int16_t>::max)(), "MAGIC_ENUM_RANGE_MAX must be less than INT16_MAX.");++ static_assert(MAGIC_ENUM_RANGE_MAX > MAGIC_ENUM_RANGE_MIN, "MAGIC_ENUM_RANGE_MAX must be greater than MAGIC_ENUM_RANGE_MIN.");++// If need cunstom names for enum, add specialization enum_name for necessary enum type.+ template <typename E>+ constexpr string_view enum_name(E) noexcept {+ static_assert(std::is_enum_v<E>, "magic_enum::customize::enum_name requires enum type.");++ return {};+ }++ } // namespace magic_enum::customize++ namespace detail {++ template <typename T>+ struct supported+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED || defined(MAGIC_ENUM_NO_CHECK_SUPPORT)+ : std::true_type {};+#else+ : std::false_type {};+#endif++ struct char_equal_to {+ constexpr bool operator()(char lhs, char rhs) const noexcept {+ return lhs == rhs;+ }+ };++ template <std::size_t N>+ class static_string {+ public:+ constexpr explicit static_string(string_view str) noexcept : static_string{str, std::make_index_sequence<N>{}} {+ assert(str.size() == N);+ }++ constexpr const char* data() const noexcept { return chars_.data(); }++ constexpr std::size_t size() const noexcept { return N; }++ constexpr operator string_view() const noexcept { return {data(), size()}; }++ private:+ template <std::size_t... I>+ constexpr static_string(string_view str, std::index_sequence<I...>) noexcept : chars_{{str[I]..., '\0'}} {}++ const std::array<char, N + 1> chars_;+ };++ template <>+ class static_string<0> {+ public:+ constexpr explicit static_string(string_view) noexcept {}++ constexpr const char* data() const noexcept { return nullptr; }++ constexpr std::size_t size() const noexcept { return 0; }++ constexpr operator string_view() const noexcept { return {}; }+ };++ constexpr string_view pretty_name(string_view name) noexcept {+ for (std::size_t i = name.size(); i > 0; --i) {+ if (!((name[i - 1] >= '0' && name[i - 1] <= '9') ||+ (name[i - 1] >= 'a' && name[i - 1] <= 'z') ||+ (name[i - 1] >= 'A' && name[i - 1] <= 'Z') ||+ (name[i - 1] == '_'))) {+ name.remove_prefix(i);+ break;+ }+ }++ if (name.size() > 0 && ((name.front() >= 'a' && name.front() <= 'z') ||+ (name.front() >= 'A' && name.front() <= 'Z') ||+ (name.front() == '_'))) {+ return name;+ }++ return {}; // Invalid name.+ }++ constexpr std::size_t find(string_view str, char c) noexcept {+#if defined(__clang__) && __clang_major__ < 9 && defined(__GLIBCXX__) || defined(_MSC_VER) && _MSC_VER < 1920 && !defined(__clang__)+ // https://stackoverflow.com/questions/56484834/constexpr-stdstring-viewfind-last-of-doesnt-work-on-clang-8-with-libstdc+// https://developercommunity.visualstudio.com/content/problem/360432/vs20178-regression-c-failed-in-test.html+ constexpr bool workaround = true;+#else+ constexpr bool workaround = false;+#endif+ if constexpr (workaround) {+ for (std::size_t i = 0; i < str.size(); ++i) {+ if (str[i] == c) {+ return i;+ }+ }++ return string_view::npos;+ } else {+ return str.find_first_of(c);+ }+ }++ template <typename BinaryPredicate>+ constexpr bool cmp_equal(string_view lhs, string_view rhs, BinaryPredicate&& p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) {+#if defined(_MSC_VER) && _MSC_VER < 1920 && !defined(__clang__)+ // https://developercommunity.visualstudio.com/content/problem/360432/vs20178-regression-c-failed-in-test.html+ // https://developercommunity.visualstudio.com/content/problem/232218/c-constexpr-string-view.html+ constexpr bool workaround = true;+#else+ constexpr bool workaround = false;+#endif+ constexpr bool default_predicate = std::is_same_v<std::decay_t<BinaryPredicate>, char_equal_to>;++ if constexpr (default_predicate && !workaround) {+ static_cast<void>(p);+ return lhs == rhs;+ } else {+ if (lhs.size() != rhs.size()) {+ return false;+ }++ const auto size = lhs.size();+ for (std::size_t i = 0; i < size; ++i) {+ if (!p(lhs[i], rhs[i])) {+ return false;+ }+ }++ return true;+ }+ }++ template <typename L, typename R>+ constexpr bool cmp_less(L lhs, R rhs) noexcept {+ static_assert(std::is_integral_v<L> && std::is_integral_v<R>, "magic_enum::detail::cmp_less requires integral type.");++ if constexpr (std::is_signed_v<L> == std::is_signed_v<R>) {+ // If same signedness (both signed or both unsigned).+ return lhs < rhs;+ } else if constexpr (std::is_signed_v<R>) {+ // If 'right' is negative, then result is 'false', otherwise cast & compare.+ return rhs > 0 && lhs < static_cast<std::make_unsigned_t<R>>(rhs);+ } else {+ // If 'left' is negative, then result is 'true', otherwise cast & compare.+ return lhs < 0 || static_cast<std::make_unsigned_t<L>>(lhs) < rhs;+ }+ }++ template <typename I>+ constexpr I log2(I value) noexcept {+ static_assert(std::is_integral_v<I>, "magic_enum::detail::log2 requires integral type.");++ auto ret = I{0};+ for (; value > I{1}; value >>= I{1}, ++ret) {}++ return ret;+ }++ template <typename I>+ constexpr bool is_pow2(I x) noexcept {+ static_assert(std::is_integral_v<I>, "magic_enum::detail::is_pow2 requires integral type.");++ return x != 0 && (x & (x - 1)) == 0;+ }++ template <typename T>+ inline constexpr bool is_enum_v = std::is_enum_v<T> && std::is_same_v<T, std::decay_t<T>>;++ template <typename E>+ constexpr auto n() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::n requires enum type.");+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED+# if defined(__clang__)+ constexpr string_view name{__PRETTY_FUNCTION__ + 34, sizeof(__PRETTY_FUNCTION__) - 36};+# elif defined(__GNUC__)+ constexpr string_view name{__PRETTY_FUNCTION__ + 49, sizeof(__PRETTY_FUNCTION__) - 51};+# elif defined(_MSC_VER)+ constexpr string_view name{__FUNCSIG__ + 40, sizeof(__FUNCSIG__) - 57};+# endif+ return static_string<name.size()>{name};+#else+ return string_view{}; // Unsupported compiler.+#endif+ }++ template <typename E>+ inline constexpr auto type_name_v = n<E>();++ template <typename E, E V>+ constexpr auto n() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::n requires enum type.");+ constexpr auto custom_name = customize::enum_name<E>(V);++ if constexpr (custom_name.empty()) {+ static_cast<void>(custom_name);+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED+# if defined(__clang__) || defined(__GNUC__)+ constexpr auto name = pretty_name({__PRETTY_FUNCTION__, sizeof(__PRETTY_FUNCTION__) - 2});+# elif defined(_MSC_VER)+ constexpr auto name = pretty_name({__FUNCSIG__, sizeof(__FUNCSIG__) - 17});+# endif+ return static_string<name.size()>{name};+#else+ return string_view{}; // Unsupported compiler.+#endif+ } else {+ return static_string<custom_name.size()>{custom_name};+ }+ }++ template <typename E, E V>+ inline constexpr auto enum_name_v = n<E, V>();++ template <typename E, auto V>+ constexpr bool is_valid() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::is_valid requires enum type.");++ return n<E, static_cast<E>(V)>().size() != 0;+ }++ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>+ constexpr int reflected_min() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::reflected_min requires enum type.");++ if constexpr (IsFlags) {+ return 0;+ } else {+ constexpr auto lhs = customize::enum_range<E>::min;+ static_assert(lhs > (std::numeric_limits<std::int16_t>::min)(), "magic_enum::enum_range requires min must be greater than INT16_MIN.");+ constexpr auto rhs = (std::numeric_limits<U>::min)();++ if constexpr (cmp_less(lhs, rhs)) {+ return rhs;+ } else {+ return lhs;+ }+ }+ }++ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>+ constexpr int reflected_max() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::reflected_max requires enum type.");++ if constexpr (IsFlags) {+ return std::numeric_limits<U>::digits - 1;+ } else {+ constexpr auto lhs = customize::enum_range<E>::max;+ static_assert(lhs < (std::numeric_limits<std::int16_t>::max)(), "magic_enum::enum_range requires max must be less than INT16_MAX.");+ constexpr auto rhs = (std::numeric_limits<U>::max)();++ if constexpr (cmp_less(lhs, rhs)) {+ return lhs;+ } else {+ return rhs;+ }+ }+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto reflected_min_v = reflected_min<E, IsFlags>();++ template <typename E, bool IsFlags = false>+ inline constexpr auto reflected_max_v = reflected_max<E, IsFlags>();++ template <typename E, int O, bool IsFlags = false, typename U = std::underlying_type_t<E>>+ constexpr E value(std::size_t i) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::value requires enum type.");++ if constexpr (IsFlags) {+ return static_cast<E>(U{1} << static_cast<U>(static_cast<int>(i) + O));+ } else {+ return static_cast<E>(static_cast<int>(i) + O);+ }+ }++ template <std::size_t N>+ constexpr std::size_t values_count(const std::array<bool, N>& valid) noexcept {+ auto count = std::size_t{0};+ for (std::size_t i = 0; i < valid.size(); ++i) {+ if (valid[i]) {+ ++count;+ }+ }++ return count;+ }++ template <typename E, bool IsFlags, int Min, std::size_t... I>+ constexpr auto values(std::index_sequence<I...>) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::values requires enum type.");+ constexpr std::array<bool, sizeof...(I)> valid{{is_valid<E, value<E, Min, IsFlags>(I)>()...}};+ constexpr std::size_t count = values_count(valid);++ std::array<E, count> values{};+ for (std::size_t i = 0, v = 0; v < count; ++i) {+ if (valid[i]) {+ values[v++] = value<E, Min, IsFlags>(i);+ }+ }++ return values;+ }++ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>+ constexpr auto values() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::values requires enum type.");+ constexpr auto min = reflected_min_v<E, IsFlags>;+ constexpr auto max = reflected_max_v<E, IsFlags>;+ constexpr auto range_size = max - min + 1;+ static_assert(range_size > 0, "magic_enum::enum_range requires valid size.");+ static_assert(range_size < (std::numeric_limits<std::uint16_t>::max)(), "magic_enum::enum_range requires valid size.");+ if constexpr (cmp_less((std::numeric_limits<U>::min)(), min) && !IsFlags) {+ static_assert(!is_valid<E, value<E, min - 1, IsFlags>(0)>(), "magic_enum::enum_range detects enum value smaller than min range size.");+ }+ if constexpr (cmp_less(range_size, (std::numeric_limits<U>::max)()) && !IsFlags) {+ static_assert(!is_valid<E, value<E, min, IsFlags>(range_size + 1)>(), "magic_enum::enum_range detects enum value larger than max range size.");+ }++ return values<E, IsFlags, reflected_min_v<E, IsFlags>>(std::make_index_sequence<range_size>{});+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto values_v = values<E, IsFlags>();++ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>+ using values_t = decltype((values_v<D, IsFlags>));++ template <typename E, bool IsFlags = false>+ inline constexpr auto count_v = values_v<E, IsFlags>.size();++ template <typename E, bool IsFlags = false, typename U = std::underlying_type_t<E>>+ inline constexpr auto min_v = static_cast<U>(values_v<E, IsFlags>.front());++ template <typename E, bool IsFlags = false, typename U = std::underlying_type_t<E>>+ inline constexpr auto max_v = static_cast<U>(values_v<E, IsFlags>.back());++ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>+ constexpr std::size_t range_size() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::range_size requires enum type.");+ constexpr auto max = IsFlags ? log2(max_v<E, IsFlags>) : max_v<E, IsFlags>;+ constexpr auto min = IsFlags ? log2(min_v<E, IsFlags>) : min_v<E, IsFlags>;+ constexpr auto range_size = max - min + U{1};+ static_assert(range_size > 0, "magic_enum::enum_range requires valid size.");+ static_assert(range_size < (std::numeric_limits<std::uint16_t>::max)(), "magic_enum::enum_range requires valid size.");++ return static_cast<std::size_t>(range_size);+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto range_size_v = range_size<E, IsFlags>();++ template <typename E, bool IsFlags = false>+ using index_t = std::conditional_t<range_size_v<E, IsFlags> < (std::numeric_limits<std::uint8_t>::max)(), std::uint8_t, std::uint16_t>;++ template <typename E, bool IsFlags = false>+ inline constexpr auto invalid_index_v = (std::numeric_limits<index_t<E, IsFlags>>::max)();++ template <typename E, bool IsFlags, std::size_t... I>+ constexpr auto indexes(std::index_sequence<I...>) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::indexes requires enum type.");+ constexpr auto min = IsFlags ? log2(min_v<E, IsFlags>) : min_v<E, IsFlags>;+ [[maybe_unused]] auto i = index_t<E, IsFlags>{0};++ return std::array<decltype(i), sizeof...(I)>{{(is_valid<E, value<E, min, IsFlags>(I)>() ? i++ : invalid_index_v<E, IsFlags>)...}};+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto indexes_v = indexes<E, IsFlags>(std::make_index_sequence<range_size_v<E, IsFlags>>{});++ template <typename E, bool IsFlags, std::size_t... I>+ constexpr auto names(std::index_sequence<I...>) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::names requires enum type.");++ return std::array<string_view, sizeof...(I)>{{enum_name_v<E, values_v<E, IsFlags>[I]>...}};+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto names_v = names<E, IsFlags>(std::make_index_sequence<count_v<E, IsFlags>>{});++ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>+ using names_t = decltype((names_v<D, IsFlags>));++ template <typename E, bool IsFlags, std::size_t... I>+ constexpr auto entries(std::index_sequence<I...>) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::entries requires enum type.");++ return std::array<std::pair<E, string_view>, sizeof...(I)>{{{values_v<E, IsFlags>[I], enum_name_v<E, values_v<E, IsFlags>[I]>}...}};+ }++ template <typename E, bool IsFlags = false>+ inline constexpr auto entries_v = entries<E, IsFlags>(std::make_index_sequence<count_v<E, IsFlags>>{});++ template <typename E, bool IsFlags = false, typename D = std::decay_t<E>>+ using entries_t = decltype((entries_v<D, IsFlags>));++ template <typename E, bool IsFlags, typename U = std::underlying_type_t<E>>+ constexpr bool is_sparse() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::is_sparse requires enum type.");++ return range_size_v<E, IsFlags> != count_v<E, IsFlags>;+ }++ template <typename E, bool IsFlags = false>+ inline constexpr bool is_sparse_v = is_sparse<E, IsFlags>();++ template <typename E, typename U = std::underlying_type_t<E>>+ constexpr std::size_t undex(U value) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::undex requires enum type.");++ if (const auto i = static_cast<std::size_t>(value - min_v<E>); value >= min_v<E> && value <= max_v<E>) {+ if constexpr (is_sparse_v<E>) {+ if (const auto idx = indexes_v<E>[i]; idx != invalid_index_v<E>) {+ return idx;+ }+ } else {+ return i;+ }+ }++ return invalid_index_v<E>; // Value out of range.+ }++ template <typename E, typename U = std::underlying_type_t<E>>+ constexpr std::size_t endex(E value) noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::endex requires enum type.");++ return undex<E>(static_cast<U>(value));+ }++ template <typename E, typename U = std::underlying_type_t<E>>+ constexpr U value_ors() noexcept {+ static_assert(is_enum_v<E>, "magic_enum::detail::endex requires enum type.");++ auto value = U{0};+ for (std::size_t i = 0; i < count_v<E, true>; ++i) {+ value |= static_cast<U>(values_v<E, true>[i]);+ }++ return value;+ }++ template <bool, typename T, typename R>+ struct enable_if_enum {};++ template <typename T, typename R>+ struct enable_if_enum<true, T, R> {+ using type = R;+ using D = std::decay_t<T>;+ static_assert(supported<D>::value, "magic_enum unsupported compiler (https://github.com/Neargye/magic_enum#compiler-compatibility).");+ };++ template <typename T, typename R = void>+ using enable_if_enum_t = std::enable_if_t<std::is_enum_v<std::decay_t<T>>, R>;++ template <typename T, typename Enable = std::enable_if_t<std::is_enum_v<std::decay_t<T>>>>+ using enum_concept = T;++ template <typename T, bool = std::is_enum_v<T>>+ struct is_scoped_enum : std::false_type {};++ template <typename T>+ struct is_scoped_enum<T, true> : std::bool_constant<!std::is_convertible_v<T, std::underlying_type_t<T>>> {};++ template <typename T, bool = std::is_enum_v<T>>+ struct is_unscoped_enum : std::false_type {};++ template <typename T>+ struct is_unscoped_enum<T, true> : std::bool_constant<std::is_convertible_v<T, std::underlying_type_t<T>>> {};++ template <typename T, bool = std::is_enum_v<std::decay_t<T>>>+ struct underlying_type {};++ template <typename T>+ struct underlying_type<T, true> : std::underlying_type<std::decay_t<T>> {};++ } // namespace magic_enum::detail++// Checks is magic_enum supported compiler.+ inline constexpr bool is_magic_enum_supported = detail::supported<void>::value;++ template <typename T>+ using Enum = detail::enum_concept<T>;++// Checks whether T is an Unscoped enumeration type.+// Provides the member constant value which is equal to true, if T is an [Unscoped enumeration](https://en.cppreference.com/w/cpp/language/enum#Unscoped_enumeration) type. Otherwise, value is equal to false.+ template <typename T>+ struct is_unscoped_enum : detail::is_unscoped_enum<T> {};++ template <typename T>+ inline constexpr bool is_unscoped_enum_v = is_unscoped_enum<T>::value;++// Checks whether T is an Scoped enumeration type.+// Provides the member constant value which is equal to true, if T is an [Scoped enumeration](https://en.cppreference.com/w/cpp/language/enum#Scoped_enumerations) type. Otherwise, value is equal to false.+ template <typename T>+ struct is_scoped_enum : detail::is_scoped_enum<T> {};++ template <typename T>+ inline constexpr bool is_scoped_enum_v = is_scoped_enum<T>::value;++// If T is a complete enumeration type, provides a member typedef type that names the underlying type of T.+// Otherwise, if T is not an enumeration type, there is no member type. Otherwise (T is an incomplete enumeration type), the program is ill-formed.+ template <typename T>+ struct underlying_type : detail::underlying_type<T> {};++ template <typename T>+ using underlying_type_t = typename underlying_type<T>::type;++// Returns type name of enum.+ template <typename E>+ [[nodiscard]] constexpr auto enum_type_name() noexcept -> detail::enable_if_enum_t<E, string_view> {+ using D = std::decay_t<E>;+ constexpr string_view name = detail::type_name_v<D>;+ static_assert(name.size() > 0, "Enum type does not have a name.");++ return name;+ }++// Returns number of enum values.+ template <typename E>+ [[nodiscard]] constexpr auto enum_count() noexcept -> detail::enable_if_enum_t<E, std::size_t> {+ using D = std::decay_t<E>;++ return detail::count_v<D>;+ }++// Returns enum value at specified index.+// No bounds checking is performed: the behavior is undefined if index >= number of enum values.+ template <typename E>+ [[nodiscard]] constexpr auto enum_value(std::size_t index) noexcept -> detail::enable_if_enum_t<E, std::decay_t<E>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");++ if constexpr (detail::is_sparse_v<D>) {+ return assert((index < detail::count_v<D>)), detail::values_v<D>[index];+ } else {+ return assert((index < detail::count_v<D>)), detail::value<D, detail::min_v<D>>(index);+ }+ }++// Returns std::array with enum values, sorted by enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_values() noexcept -> detail::enable_if_enum_t<E, detail::values_t<E>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");++ return detail::values_v<D>;+ }++// Returns name from static storage enum variable.+// This version is much lighter on the compile times and is not restricted to the enum_range limitation.+ template <auto V>+ [[nodiscard]] constexpr auto enum_name() noexcept -> detail::enable_if_enum_t<decltype(V), string_view> {+ using D = std::decay_t<decltype(V)>;+ constexpr string_view name = detail::enum_name_v<D, V>;+ static_assert(name.size() > 0, "Enum value does not have a name.");++ return name;+ }++// Returns name from enum value.+// If enum value does not have name or value out of range, returns empty string.+ template <typename E>+ [[nodiscard]] constexpr auto enum_name(E value) noexcept -> detail::enable_if_enum_t<E, string_view> {+ using D = std::decay_t<E>;++ if (const auto i = detail::endex<D>(value); i != detail::invalid_index_v<D>) {+ return detail::names_v<D>[i];+ }++ return {}; // Invalid value or out of range.+ }++// Returns std::array with names, sorted by enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_names() noexcept -> detail::enable_if_enum_t<E, detail::names_t<E>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");++ return detail::names_v<D>;+ }++// Returns std::array with pairs (value, name), sorted by enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_entries() noexcept -> detail::enable_if_enum_t<E, detail::entries_t<E>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D> > 0, "magic_enum requires enum implementation and valid max and min.");++ return detail::entries_v<D>;+ }++// Obtains enum value from integer value.+// Returns optional with enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_cast(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ using D = std::decay_t<E>;++ if (detail::undex<D>(value) != detail::invalid_index_v<D>) {+ return static_cast<D>(value);+ }++ return {}; // Invalid value or out of range.+ }++// Obtains enum value from name.+// Returns optional with enum value.+ template <typename E, typename BinaryPredicate>+ [[nodiscard]] constexpr auto enum_cast(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::enum_cast requires bool(char, char) invocable predicate.");+ using D = std::decay_t<E>;++ for (std::size_t i = 0; i < detail::count_v<D>; ++i) {+ if (detail::cmp_equal(value, detail::names_v<D>[i], p)) {+ return enum_value<D>(i);+ }+ }++ return {}; // Invalid value or out of range.+ }++// Obtains enum value from name.+// Returns optional with enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_cast(string_view value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ using D = std::decay_t<E>;++ return enum_cast<D>(value, detail::char_equal_to{});+ }++// Returns integer value from enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_integer(E value) noexcept -> detail::enable_if_enum_t<E, underlying_type_t<E>> {+ return static_cast<underlying_type_t<E>>(value);+ }++// Obtains index in enum values from enum value.+// Returns optional with index.+ template <typename E>+ [[nodiscard]] constexpr auto enum_index(E value) noexcept -> detail::enable_if_enum_t<E, optional<std::size_t>> {+ using D = std::decay_t<E>;++ if (const auto i = detail::endex<D>(value); i != detail::invalid_index_v<D>) {+ return i;+ }++ return {}; // Invalid value or out of range.+ }++// Checks whether enum contains enumerator with such enum value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(E value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;++ return detail::endex<D>(value) != detail::invalid_index_v<D>;+ }++// Checks whether enum contains enumerator with such integer value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;++ return detail::undex<D>(value) != detail::invalid_index_v<D>;+ }++// Checks whether enum contains enumerator with such name.+ template <typename E, typename BinaryPredicate>+ [[nodiscard]] constexpr auto enum_contains(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, bool> {+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::enum_contains requires bool(char, char) invocable predicate.");+ using D = std::decay_t<E>;++ return enum_cast<D>(value, std::move_if_noexcept(p)).has_value();+ }++// Checks whether enum contains enumerator with such name.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(string_view value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;++ return enum_cast<D>(value).has_value();+ }++ namespace ostream_operators {++ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, E value) {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED+ if (const auto name = magic_enum::enum_name<D>(value); !name.empty()) {+ for (const auto c : name) {+ os.put(c);+ }+ return os;+ }+#endif+ return (os << static_cast<U>(value));+ }++ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, optional<E> value) {+ return value.has_value() ? (os << value.value()) : os;+ }++ } // namespace magic_enum::ostream_operators++ namespace bitwise_operators {++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E operator~(E rhs) noexcept {+ return static_cast<E>(~static_cast<underlying_type_t<E>>(rhs));+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E operator|(E lhs, E rhs) noexcept {+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) | static_cast<underlying_type_t<E>>(rhs));+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E operator&(E lhs, E rhs) noexcept {+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) & static_cast<underlying_type_t<E>>(rhs));+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E operator^(E lhs, E rhs) noexcept {+ return static_cast<E>(static_cast<underlying_type_t<E>>(lhs) ^ static_cast<underlying_type_t<E>>(rhs));+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E& operator|=(E& lhs, E rhs) noexcept {+ return lhs = (lhs | rhs);+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E& operator&=(E& lhs, E rhs) noexcept {+ return lhs = (lhs & rhs);+ }++ template <typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ constexpr E& operator^=(E& lhs, E rhs) noexcept {+ return lhs = (lhs ^ rhs);+ }++ } // namespace magic_enum::bitwise_operators++ namespace flags {++// Returns type name of enum.+ using magic_enum::enum_type_name;++// Returns number of enum-flags values.+ template <typename E>+ [[nodiscard]] constexpr auto enum_count() noexcept -> detail::enable_if_enum_t<E, std::size_t> {+ using D = std::decay_t<E>;++ return detail::count_v<D, true>;+ }++// Returns enum-flags value at specified index.+// No bounds checking is performed: the behavior is undefined if index >= number of enum-flags values.+ template <typename E>+ [[nodiscard]] constexpr auto enum_value(std::size_t index) noexcept -> detail::enable_if_enum_t<E, std::decay_t<E>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");++ if constexpr (detail::is_sparse_v<D, true>) {+ return assert((index < detail::count_v<D, true>)), detail::values_v<D, true>[index];+ } else {+ constexpr auto min = detail::log2(detail::min_v<D, true>);++ return assert((index < detail::count_v<D, true>)), detail::value<D, min, true>(index);+ }+ }++// Returns std::array with enum-flags values, sorted by enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_values() noexcept -> detail::enable_if_enum_t<E, detail::values_t<E, true>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");++ return detail::values_v<D, true>;+ }++// Returns name from enum-flags value.+// If enum-flags value does not have name or value out of range, returns empty string.+ template <typename E>+ [[nodiscard]] auto enum_name(E value) -> detail::enable_if_enum_t<E, string> {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;++ string name;+ auto check_value = U{0};+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {+ if (const auto v = static_cast<U>(enum_value<D>(i)); (static_cast<U>(value) & v) != 0) {+ check_value |= v;+ const auto n = detail::names_v<D, true>[i];+ if (!name.empty()) {+ name.append(1, '|');+ }+ name.append(n.data(), n.size());+ }+ }++ if (check_value != 0 && check_value == static_cast<U>(value)) {+ return name;+ }++ return {}; // Invalid value or out of range.+ }++// Returns std::array with string names, sorted by enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_names() noexcept -> detail::enable_if_enum_t<E, detail::names_t<E, true>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");++ return detail::names_v<D, true>;+ }++// Returns std::array with pairs (value, name), sorted by enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_entries() noexcept -> detail::enable_if_enum_t<E, detail::entries_t<E, true>> {+ using D = std::decay_t<E>;+ static_assert(detail::count_v<D, true> > 0, "magic_enum::flags requires enum-flags implementation.");++ return detail::entries_v<D, true>;+ }++// Obtains enum-flags value from integer value.+// Returns optional with enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_cast(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;++ if constexpr (detail::is_sparse_v<D, true>) {+ auto check_value = U{0};+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {+ if (const auto v = static_cast<U>(enum_value<D>(i)); (value & v) != 0) {+ check_value |= v;+ }+ }++ if (check_value != 0 && check_value == value) {+ return static_cast<D>(value);+ }+ } else {+ constexpr auto min = detail::min_v<D, true>;+ constexpr auto max = detail::value_ors<D>();++ if (value >= min && value <= max) {+ return static_cast<D>(value);+ }+ }++ return {}; // Invalid value or out of range.+ }++// Obtains enum-flags value from name.+// Returns optional with enum-flags value.+ template <typename E, typename BinaryPredicate>+ [[nodiscard]] constexpr auto enum_cast(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::flags::enum_cast requires bool(char, char) invocable predicate.");+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;++ auto result = U{0};+ while (!value.empty()) {+ const auto d = detail::find(value, '|');+ const auto s = (d == string_view::npos) ? value : value.substr(0, d);+ auto f = U{0};+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {+ if (detail::cmp_equal(s, detail::names_v<D, true>[i], p)) {+ f = static_cast<U>(enum_value<D>(i));+ result |= f;+ break;+ }+ }+ if (f == U{0}) {+ return {}; // Invalid value or out of range.+ }+ value.remove_prefix((d == string_view::npos) ? value.size() : d + 1);+ }++ if (result == U{0}) {+ return {}; // Invalid value or out of range.+ } else {+ return static_cast<D>(result);+ }+ }++// Obtains enum-flags value from name.+// Returns optional with enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_cast(string_view value) noexcept -> detail::enable_if_enum_t<E, optional<std::decay_t<E>>> {+ using D = std::decay_t<E>;++ return enum_cast<D>(value, detail::char_equal_to{});+ }++// Returns integer value from enum value.+ using magic_enum::enum_integer;++// Obtains index in enum-flags values from enum-flags value.+// Returns optional with index.+ template <typename E>+ [[nodiscard]] constexpr auto enum_index(E value) noexcept -> detail::enable_if_enum_t<E, optional<std::size_t>> {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;++ if (detail::is_pow2(static_cast<U>(value))) {+ for (std::size_t i = 0; i < detail::count_v<D, true>; ++i) {+ if (enum_value<D>(i) == value) {+ return i;+ }+ }+ }++ return {}; // Invalid value or out of range.+ }++// Checks whether enum-flags contains enumerator with such enum-flags value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(E value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;++ return enum_cast<D>(static_cast<U>(value)).has_value();+ }++// Checks whether enum-flags contains enumerator with such integer value.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(underlying_type_t<E> value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;++ return enum_cast<D>(value).has_value();+ }++// Checks whether enum-flags contains enumerator with such name.+ template <typename E, typename BinaryPredicate>+ [[nodiscard]] constexpr auto enum_contains(string_view value, BinaryPredicate p) noexcept(std::is_nothrow_invocable_r_v<bool, BinaryPredicate, char, char>) -> detail::enable_if_enum_t<E, bool> {+ static_assert(std::is_invocable_r_v<bool, BinaryPredicate, char, char>, "magic_enum::flags::enum_contains requires bool(char, char) invocable predicate.");+ using D = std::decay_t<E>;++ return enum_cast<D>(value, std::move_if_noexcept(p)).has_value();+ }++// Checks whether enum-flags contains enumerator with such name.+ template <typename E>+ [[nodiscard]] constexpr auto enum_contains(string_view value) noexcept -> detail::enable_if_enum_t<E, bool> {+ using D = std::decay_t<E>;++ return enum_cast<D>(value).has_value();+ }++ } // namespace magic_enum::flags++ namespace flags::ostream_operators {++ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, E value) {+ using D = std::decay_t<E>;+ using U = underlying_type_t<D>;+#if defined(MAGIC_ENUM_SUPPORTED) && MAGIC_ENUM_SUPPORTED+ if (const auto name = magic_enum::flags::enum_name<D>(value); !name.empty()) {+ for (const auto c : name) {+ os.put(c);+ }+ return os;+ }+#endif+ return (os << static_cast<U>(value));+ }++ template <typename Char, typename Traits, typename E, std::enable_if_t<std::is_enum_v<E>, int> = 0>+ std::basic_ostream<Char, Traits>& operator<<(std::basic_ostream<Char, Traits>& os, optional<E> value) {+ return value.has_value() ? (os << value.value()) : os;+ }++ } // namespace magic_enum::flags::ostream_operators++ namespace flags::bitwise_operators {++ using namespace magic_enum::bitwise_operators;++ } // namespace magic_enum::flags::bitwise_operators++} // namespace magic_enum++#if defined(__clang__)+# pragma clang diagnostic pop+#elif defined(__GNUC__)+# pragma GCC diagnostic pop+#elif defined(_MSC_VER)+# pragma warning(pop)+#endif++#endif // NEARGYE_MAGIC_ENUM_HPPdiff --git a/include/string_utils.h b/include/string_utils.hindex 7f641527..adbec688 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -294,4 +294,8 @@ struct StringUtils {static std::string hash_sha256(const std::string& str);//static size_t unicode_length(const std::string& bytes);++ static bool begins_with(const std::string& str, const std::string& prefix) {+ return str.rfind(prefix, 0) == 0;+ }};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 376f20a5..91b29bed 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -160,10 +160,11 @@ nlohmann::json Collection::get_summary_json() const {}Option<nlohmann::json> Collection::add(const std::string & json_str,- const index_operation_t& operation, const std::string& id) {+ const index_operation_t& operation, const std::string& id,+ const DIRTY_VALUES& dirty_values) {nlohmann::json document;std::vector<std::string> json_lines = {json_str};- const nlohmann::json& res = add_many(json_lines, document, operation, id);+ const nlohmann::json& res = add_many(json_lines, document, operation, id, dirty_values);if(!res["success"].get<bool>()) {nlohmann::json res_doc;@@ -203,7 +204,8 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json}nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmann::json& document,- const index_operation_t& operation, const std::string& id) {+ const index_operation_t& operation, const std::string& id,+ const DIRTY_VALUES& dirty_values) {//LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();std::vector<std::vector<index_record>> iter_batch;@@ -220,7 +222,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaOption<doc_seq_id_t> doc_seq_id_op = to_doc(json_line, document, operation, id);const uint32_t seq_id = doc_seq_id_op.ok() ? doc_seq_id_op.get().seq_id : 0;- index_record record(i, seq_id, document, operation);+ index_record record(i, seq_id, document, operation, dirty_values);// NOTE: we overwrite the input json_lines with result to avoid memory pressure@@ -237,62 +239,9 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma// if `index_all_fields` is enabled, we will have to update schema first before indexingif(index_all_fields) {- size_t old_field_count = fields.size();- for(const auto& kv: document.items()) {- if (search_schema.count(kv.key()) == 0) {- std::string field_type;- bool parseable = field::get_type(kv.value(), field_type);- if (parseable) {- // this ensures that we can handle the same field having different types in different docs- const std::string fname = kv.key();// + "_" + field_type;- field new_field(fname, field_type, false);- search_schema.emplace(fname, new_field);- fields.emplace_back(new_field);- }- }- }-- if(fields.size() != old_field_count) {- // we should persist changes to fields in store- std::string coll_meta_json;- StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);-- if(status != StoreStatus::FOUND) {- record.index_failure(500, "Could not fetch collection meta from store.");- continue;- }-- nlohmann::json collection_meta;-- try {- collection_meta = nlohmann::json::parse(coll_meta_json);- bool found_default_sorting_field = false;- nlohmann::json fields_json = nlohmann::json::array();;-- Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json,- found_default_sorting_field);-- if(!fields_json_op.ok()) {- record.index_failure(fields_json_op.code(), fields_json_op.error());- continue;- }-- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;- bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());-- if(!persisted) {- record.index_failure(500, "Could not persist collection meta to store.");- continue;- }-- for(auto index: indices) {- index->refresh_search_schema(search_schema);- }-- } catch(...) {- record.index_failure(500, "Unable to parse collection meta.");- continue;- }+ Option<bool> schema_change_op = check_and_update_schema(document);+ if(!schema_change_op.ok()) {+ record.index_failure(schema_change_op.code(), schema_change_op.error());}}}@@ -353,7 +302,7 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batchif(!write_ok) {// we will attempt to reindex the old doc on a best-effort basisremove_document(index_record.new_doc, index_record.seq_id, false);- index_in_memory(index_record.old_doc, index_record.seq_id, false);+ index_in_memory(index_record.old_doc, index_record.seq_id, false, index_record.dirty_values);index_record.index_failure(500, "Could not write to on-disk storage.");} else {num_indexed++;@@ -398,13 +347,15 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch}}-Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id, bool is_update) {+Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t seq_id,+ bool is_update, const DIRTY_VALUES& dirty_values) {std::unique_lock lock(mutex);if(!is_update) {- // for update, validation should be done prior+ // for non-update, validation should be done priorOption<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,- search_schema, facet_schema, is_update);+ search_schema, facet_schema, is_update,+ index_all_fields, dirty_values);if(!validation_op.ok()) {return validation_op;@@ -433,7 +384,7 @@ size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> &CollectionManager::get_instance().get_thread_pool()->enqueue([index, index_id, &num_indexed_vec, &iter_batch, this, &m_process, &num_processed, &cv_process]() {size_t num_indexed = Index::batch_memory_index(index, std::ref(iter_batch[index_id]), default_sorting_field,- search_schema, facet_schema);+ search_schema, facet_schema, index_all_fields);std::unique_lock<std::mutex> lock(m_process);num_indexed_vec[index_id] = num_indexed;num_processed++;@@ -2277,6 +2228,71 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {return synonym_definitions;}+Option<bool> Collection::check_and_update_schema(nlohmann::json& document) {+ std::unique_lock lock(mutex);++ std::vector<field> new_fields;+ for(const auto& kv: document.items()) {+ // we will not index the special "id" key+ if (search_schema.count(kv.key()) == 0 && kv.key() != "id") {+ std::string field_type;+ bool parseable = field::get_type(kv.value(), field_type);+ if (parseable) {+ const std::string& fname = kv.key();+ field new_field(fname, field_type, false);+ search_schema.emplace(fname, new_field);+ fields.emplace_back(new_field);+ new_fields.emplace_back(new_field);++ if(new_field.is_sortable()) {+ sort_schema.emplace(new_field.name, new_field);+ }+ }+ }+ }++ if(!new_fields.empty()) {+ // we should persist changes to fields in store+ std::string coll_meta_json;+ StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);++ if(status != StoreStatus::FOUND) {+ return Option<bool>(500, "Could not fetch collection meta from store.");+ }++ nlohmann::json collection_meta;++ try {+ collection_meta = nlohmann::json::parse(coll_meta_json);+ bool found_default_sorting_field = false;+ nlohmann::json fields_json = nlohmann::json::array();;++ Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json,+ found_default_sorting_field);++ if(!fields_json_op.ok()) {+ return Option<bool>(fields_json_op.code(), fields_json_op.error());+ }++ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;+ bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());++ if(!persisted) {+ return Option<bool>(500, "Could not persist collection meta to store.");+ }++ for(auto index: indices) {+ index->refresh_schemas(new_fields);+ }++ } catch(...) {+ return Option<bool>(500, "Unable to parse collection meta.");+ }+ }++ return Option<bool>(true);+}+std::vector<Index *> Collection::init_indices() {std::vector<Index*> index_list;@@ -2287,7 +2303,7 @@ std::vector<Index *> Collection::init_indices() {facet_schema.emplace(field.name, field);}- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool() || field.is_geopoint()) {+ if(field.is_sortable()) {sort_schema.emplace(field.name, field);}}@@ -2299,3 +2315,17 @@ std::vector<Index *> Collection::init_indices() {return index_list;}++DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) const {+ StringUtils::toupper(dirty_values);+ auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(dirty_values);+ DIRTY_VALUES dirty_values_action;++ if(dirty_values_op.has_value()) {+ dirty_values_action = dirty_values_op.value();+ } else {+ dirty_values_action = index_all_fields ? DIRTY_VALUES::COERCE_OR_IGNORE : DIRTY_VALUES::REJECT;+ }++ return dirty_values_action;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b9b288d6..ba6e9447 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -189,6 +189,9 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {auto begin = std::chrono::high_resolution_clock::now();+ std::string default_dirty_values = "";+ auto dirty_values = collection->parse_dirty_values_option(default_dirty_values);+while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {num_found_docs++;const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());@@ -203,7 +206,8 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {}num_valid_docs++;- iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE));++ iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));// Peek and check for last record right here so that we handle batched indexing correctly// Without doing this, the "last batch" would have to be indexed outside the loop.diff --git a/src/core_api.cpp b/src/core_api.cppindex b2070f01..d37da108 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -100,8 +100,8 @@ bool post_create_collection(http_req & req, http_res & res) {req_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;}- if(req_json.count("fields") == 0) {- res.set_400("Parameter `fields` is required.");+ if(req_json.count("fields") == 0 && req_json.count(INDEX_ALL_FIELDS) == 0) {+ res.set_400("Parameter `fields` or `index_all_fields` is required.");return false;}@@ -436,6 +436,7 @@ bool post_import_documents(http_req& req, http_res& res) {//LOG(INFO) << "req.first_chunk=" << req.first_chunk_aggregate << ", last_chunk=" << req.last_chunk_aggregate;const char *BATCH_SIZE = "batch_size";const char *ACTION = "action";+ const char *DIRTY_VALUES = "dirty_values";if(req.params.count(BATCH_SIZE) == 0) {req.params[BATCH_SIZE] = "40";@@ -445,6 +446,10 @@ bool post_import_documents(http_req& req, http_res& res) {req.params[ACTION] = "create";}+ if(req.params.count(DIRTY_VALUES) == 0) {+ req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`+ }+if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {res.final = true;res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");@@ -539,7 +544,10 @@ bool post_import_documents(http_req& req, http_res& res) {if(!single_partial_record_body) {nlohmann::json document;- nlohmann::json json_res = collection->add_many(json_lines, document, operation);++ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);+ nlohmann::json json_res = collection->add_many(json_lines, document, operation, "",+ dirty_values);//const std::string& import_summary_json = json_res.dump();//response_stream << import_summary_json << "\n";@@ -570,6 +578,8 @@ bool post_import_documents(http_req& req, http_res& res) {bool post_add_document(http_req & req, http_res & res) {const char *ACTION = "action";+ const char *DIRTY_VALUES = "dirty_values";+if(req.params.count(ACTION) == 0) {req.params[ACTION] = "create";}@@ -579,6 +589,10 @@ bool post_add_document(http_req & req, http_res & res) {return false;}+ if(req.params.count(DIRTY_VALUES) == 0) {+ req.params[DIRTY_VALUES] = ""; // set it empty as default will depend on `index_all_fields`+ }+CollectionManager & collectionManager = CollectionManager::get_instance();auto collection = collectionManager.get_collection(req.params["collection"]);@@ -588,7 +602,9 @@ bool post_add_document(http_req & req, http_res & res) {}const index_operation_t operation = get_index_operation(req.params[ACTION]);- Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation);+ const auto& dirty_values = collection->parse_dirty_values_option(req.params[DIRTY_VALUES]);++ Option<nlohmann::json> inserted_doc_op = collection->add(req.body, operation, "", dirty_values);if(!inserted_doc_op.ok()) {res.set(inserted_doc_op.code(), inserted_doc_op.error());diff --git a/src/index.cpp b/src/index.cppindex 2333fee7..ecacf9df 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -122,7 +122,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tfor(const auto& field_pair: search_schema) {const std::string & field_name = field_pair.first;- if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {+ if(document.count(field_name) == 0) {continue;}@@ -258,11 +258,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_treturn Option<>(201);}-Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,+Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,const std::map<std::string, field> & facet_schema,- bool is_update) {+ bool is_update,+ bool index_all_fields,+ const DIRTY_VALUES& dirty_values) {bool has_default_sort_field = (document.count(default_sorting_field) != 0);@@ -284,7 +286,7 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,for(const auto& field_pair: search_schema) {const std::string& field_name = field_pair.first;- if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {+ if((field_pair.second.optional || is_update || index_all_fields) && document.count(field_name) == 0) {continue;}@@ -293,79 +295,83 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,"but is not found in the document.");}- if(field_pair.second.type == field_types::STRING) {- if(!document[field_name].is_string()) {- return Option<>(400, "Field `" + field_name + "` must be a string.");+ if(field_pair.second.type == field_types::STRING && !document[field_name].is_string()) {+ Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, -1);+ if(!coerce_op.ok()) {+ return coerce_op;}} else if(field_pair.second.type == field_types::INT32) {if(!document[field_name].is_number_integer()) {- return Option<>(400, "Field `" + field_name + "` must be an int32.");+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, -1);+ if(!coerce_op.ok()) {+ return coerce_op;+ }}if(document[field_name].get<int64_t>() > INT32_MAX) {- return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");- }- } else if(field_pair.second.type == field_types::INT64) {- if(!document[field_name].is_number_integer()) {- return Option<>(400, "Field `" + field_name + "` must be an int64.");- }- } else if(field_pair.second.type == field_types::FLOAT) {- if(!document[field_name].is_number()) { // allows integer to be passed to a float field- return Option<>(400, "Field `" + field_name + "` must be a float.");- }- } else if(field_pair.second.type == field_types::BOOL) {- if(!document[field_name].is_boolean()) {- return Option<>(400, "Field `" + field_name + "` must be a bool.");- }- } else if(field_pair.second.type == field_types::GEOPOINT) {- if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");- }- if(document[field_name].size() != 2) {- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");- }- if(!document[field_name][0].is_number() || !document[field_name][1].is_number()) {- return Option<>(400, "Field `" + field_name + "` must be in the [lat,long] format.");- }- } else if(field_pair.second.type == field_types::STRING_ARRAY) {- if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be a string array.");- }- if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {- return Option<>(400, "Field `" + field_name + "` must be a string array.");- }- } else if(field_pair.second.type == field_types::INT32_ARRAY) {- if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");+ if(dirty_values == DIRTY_VALUES::IGNORE) {+ document.erase(field_name);+ continue;+ } else {+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");+ }}-- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {- return Option<>(400, "Field `" + field_name + "` must be an int32 array.");+ } else if(field_pair.second.type == field_types::INT64 && !document[field_name].is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, false);+ if(!coerce_op.ok()) {+ return coerce_op;}- } else if(field_pair.second.type == field_types::INT64_ARRAY) {- if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");+ } else if(field_pair.second.type == field_types::FLOAT && !document[field_name].is_number()) {+ // using `is_number` allows integer to be passed to a float field+ Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, false);+ if(!coerce_op.ok()) {+ return coerce_op;}-- if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {- return Option<>(400, "Field `" + field_name + "` must be an int64 array.");+ } else if(field_pair.second.type == field_types::BOOL && !document[field_name].is_boolean()) {+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, false);+ if(!coerce_op.ok()) {+ return coerce_op;}- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {+ } else if(field_pair.second.is_array()) {if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be a float array.");+ if(dirty_values == DIRTY_VALUES::IGNORE || dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ document.erase(field_name);+ continue;+ } else {+ return Option<>(400, "Field `" + field_name + "` must be an array.");+ }}- if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {- // allows integer to be passed to a float array field- return Option<>(400, "Field `" + field_name + "` must be a float array.");- }- } else if(field_pair.second.type == field_types::BOOL_ARRAY) {- if(!document[field_name].is_array()) {- return Option<>(400, "Field `" + field_name + "` must be a bool array.");- }+ for(size_t arr_index=0; arr_index < document[field_name].size(); arr_index++) {+ const auto& item = document[field_name][arr_index];- if(document[field_name].size() > 0 && !document[field_name][0].is_boolean()) {- return Option<>(400, "Field `" + field_name + "` must be a bool array.");+ if (field_pair.second.type == field_types::STRING_ARRAY && !item.is_string()) {+ Option<uint32_t> coerce_op = coerce_string(dirty_values, document, field_name, arr_index);+ if (!coerce_op.ok()) {+ return coerce_op;+ }+ } else if (field_pair.second.type == field_types::INT32_ARRAY && !item.is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, document, field_name, arr_index);+ if (!coerce_op.ok()) {+ return coerce_op;+ }+ } else if (field_pair.second.type == field_types::INT64_ARRAY && !item.is_number_integer()) {+ Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, document, field_name, true);+ if (!coerce_op.ok()) {+ return coerce_op;+ }+ } else if (field_pair.second.type == field_types::FLOAT_ARRAY && !item.is_number()) {+ // we check for `is_number` to allow whole numbers to be passed into float fields+ Option<uint32_t> coerce_op = coerce_float(dirty_values, document, field_name, true);+ if (!coerce_op.ok()) {+ return coerce_op;+ }+ } else if (field_pair.second.type == field_types::BOOL_ARRAY && !item.is_boolean()) {+ Option<uint32_t> coerce_op = coerce_bool(dirty_values, document, field_name, true);+ if (!coerce_op.ok()) {+ return coerce_op;+ }+ }}}}@@ -451,7 +457,8 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_dosize_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_batch,const std::string & default_sorting_field,const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema) {+ const std::map<std::string, field> & facet_schema,+ bool index_all_fields) {size_t num_indexed = 0;@@ -464,7 +471,10 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record> & iter_if(index_rec.operation != DELETE) {Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,default_sorting_field,- search_schema, facet_schema, index_rec.is_update);+ search_schema, facet_schema,+ index_rec.is_update,+ index_all_fields,+ index_rec.dirty_values);if(!validation_op.ok()) {index_rec.index_failure(validation_op.code(), validation_op.error());@@ -1985,7 +1995,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}- const int64_t default_score = 0;+ const int64_t default_score = INT64_MIN;int64_t scores[3] = {0};size_t match_score_index = 0;@@ -2001,7 +2011,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16if (sort_order[0] == -1) {scores[0] = -scores[0];}- scores[1] = 0;}if(sort_fields.size() > 1) {@@ -2015,7 +2024,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16if (sort_order[1] == -1) {scores[1] = -scores[1];}- scores[2] = 0;}if(sort_fields.size() > 2) {@@ -2337,19 +2345,258 @@ const spp::sparse_hash_map<std::string, num_tree_t*>& Index::_get_numerical_indereturn numerical_index;}-void Index::refresh_search_schema(const std::unordered_map<std::string, field>& src_search_schema) {- for(const auto & fname_field: src_search_schema) {- search_schema.emplace(fname_field.first, fname_field.second);+void Index::refresh_schemas(const std::vector<field>& new_fields) {+ std::unique_lock lock(mutex);++ for(const auto & new_field: new_fields) {+ search_schema.emplace(new_field.name, new_field);- if(search_index.count(fname_field.first) == 0) {- if(fname_field.second.is_string()) {+ if(search_index.count(new_field.name) == 0) {+ if(new_field.is_string()) {art_tree *t = new art_tree;art_tree_init(t);- search_index.emplace(fname_field.first, t);+ search_index.emplace(new_field.name, t);} else {num_tree_t* num_tree = new num_tree_t;- numerical_index.emplace(fname_field.first, num_tree);+ numerical_index.emplace(new_field.name, num_tree);+ }+ }++ if(sort_index.count(new_field.name) == 0) {+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ sort_index.emplace(new_field.name, doc_to_score);+ }+ }+}++Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index) {+ std::string suffix = array_index != -1 ? "an array of" : "a";++ if(dirty_values == DIRTY_VALUES::REJECT) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");+ }++ if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ return Option<uint32_t>(200);+ }++ // we will try to coerce the value to a string++ if (document[field_name].is_number_integer()) {+ document[field_name] = std::to_string((int64_t)document[field_name]);+ }++ else if(document[field_name].is_number_float()) {+ document[field_name] = std::to_string((float)document[field_name]);+ }++ else if(document[field_name].is_boolean()) {+ document[field_name] = document[field_name] == true ? "true" : "false";+ }++ else {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);}+ } else {+ // DIRTY_FIELD_COERCE_REJECT+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");}}++ return Option<>(200);+}++Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index) {+ std::string suffix = array_index != -1 ? "an array of" : "an";++ if(dirty_values == DIRTY_VALUES::REJECT) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");+ }++ if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ return Option<uint32_t>(200);+ }++ // try to value coerce into an integer++ if(document[field_name].is_number_float()) {+ document[field_name] = static_cast<int32_t>(document[field_name].get<float>());+ }++ else if(document[field_name].is_boolean()) {+ document[field_name] = document[field_name] == true ? 1 : 0;+ }++ else if(document[field_name].is_string() && StringUtils::is_int32_t(document[field_name])) {+ document[field_name] = std::atol(document[field_name].get<std::string>().c_str());+ }++ else {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ document.erase(field_name);+ } else {+ // DIRTY_FIELD_COERCE_REJECT+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");+ }+ }++ return Option<uint32_t>(200);+}++Option<uint32_t> Index::coerce_int64_t(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index) {+ std::string suffix = array_index != -1 ? "an array of" : "an";++ if(dirty_values == DIRTY_VALUES::REJECT) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");+ }++ if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ return Option<uint32_t>(200);+ }++ // try to value coerce into an integer++ if(document[field_name].is_number_float()) {+ document[field_name] = static_cast<int64_t>(document[field_name].get<float>());+ }++ else if(document[field_name].is_boolean()) {+ document[field_name] = document[field_name] == true ? 1 : 0;+ }++ else if(document[field_name].is_string() && StringUtils::is_int64_t(document[field_name])) {+ document[field_name] = std::atoll(document[field_name].get<std::string>().c_str());+ }++ else {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ } else {+ // DIRTY_FIELD_COERCE_REJECT+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");+ }+ }++ return Option<uint32_t>(200);+}++Option<uint32_t>+Index::coerce_bool(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index) {+ std::string suffix = array_index != -1 ? "a array of" : "a";+ if(dirty_values == DIRTY_VALUES::REJECT) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");+ }++ if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ return Option<uint32_t>(200);+ }++ // try to value coerce into a bool+ if (document[field_name].is_number_integer() &&+ (document[field_name].get<int64_t>() == 1 || document[field_name].get<int64_t>() == 0)) {+ document[field_name] = document[field_name].get<int64_t>() == 1;+ }++ else if(document[field_name].is_string()) {+ std::string str_val = document[field_name].get<std::string>();+ StringUtils::tolowercase(str_val);+ if(str_val == "true") {+ document[field_name] = true;+ return Option<uint32_t>(200);+ } else if(str_val == "false") {+ document[field_name] = false;+ return Option<uint32_t>(200);+ }+ }++ else {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ } else {+ // DIRTY_FIELD_COERCE_REJECT+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");+ }+ }++ return Option<uint32_t>(200);+}++Option<uint32_t>+Index::coerce_float(const DIRTY_VALUES& dirty_values, nlohmann::json &document,+ const std::string &field_name, const int array_index) {+ std::string suffix = array_index != -1 ? "a array of" : "a";++ if(dirty_values == DIRTY_VALUES::REJECT) {+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");+ }++ if(dirty_values == DIRTY_VALUES::IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ return Option<uint32_t>(200);+ }++ // try to value coerce into a float++ if(document[field_name].is_string() && StringUtils::is_float(document[field_name])) {+ document[field_name] = std::atof(document[field_name].get<std::string>().c_str());+ }++ else if(document[field_name].is_boolean()) {+ document[field_name] = document[field_name] == true ? 1.0 : 0.0;+ }++ else {+ if(dirty_values == DIRTY_VALUES::COERCE_OR_IGNORE) {+ if(array_index == -1) {+ document.erase(field_name);+ } else {+ document[field_name].erase(document[field_name].begin() + array_index);+ }+ } else {+ // DIRTY_FIELD_COERCE_REJECT+ return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");+ }+ }++ return Option<uint32_t>(200);}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 7f5a0a41..a5e34765 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -44,7 +44,7 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, true).get();}@@ -81,12 +81,62 @@ TEST_F(CollectionAllFieldsTest, IndexDocsWithoutSchema) {ASSERT_STREQ("7", results["hits"][2]["document"]["id"].get<std::string>().c_str());// reject field with a different type than already inferred type+ // default for `index_all_fields` is `DIRTY_FIELD_COERCE_IGNORE`++ // unable to coerceauto doc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell","starring_facet":"Will Ferrell","title":"Anchorman 2: The Legend Continues"})";Option<nlohmann::json> add_op = coll1->add(doc_json);ASSERT_FALSE(add_op.ok());- ASSERT_STREQ("Field `cast` must be a string array.", add_op.error().c_str());+ ASSERT_STREQ("Field `cast` must be an array.", add_op.error().c_str());++ // coerce integer to string+ doc_json = R"({"cast": ["William Barnes"],"points": 63, "starring":"Will Ferrell",+ "starring_facet":"Will Ferrell","title": 300})";++ add_op = coll1->add(doc_json);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("300", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("300", results["hits"][0]["document"]["title"].get<std::string>().c_str());++ // with dirty values set to `DIRTY_FIELD_COERCE_IGNORE`+ // `cast` field should not be indexed into store+ doc_json = R"({"cast":"William Barnes","points":63,"starring":"Will Ferrell",+ "starring_facet":"Will Ferrell","title":"With bad cast field."})";++ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_IGNORE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("With bad cast field", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("With bad cast field.", results["hits"][0]["document"]["title"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["document"].count("cast"));++ // with dirty values set to `DIRTY_FIELD_IGNORE`+ // no coercion should happen, `title` field will just be ignored, but record indexed+ doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",+ "starring_facet":"Will Ferrell","title": 1200 })";++ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::IGNORE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("1200", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search("Jeremy Livingston", {"cast"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(0, results["hits"][0]["document"].count("title"));++ // with dirty values set to `DIRTY_FIELD_REJECT`+ doc_json = R"({"cast": ["Jeremy Livingston"],"points":63,"starring":"Will Ferrell",+ "starring_facet":"Will Ferrell","title": 1200 })";++ add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_STREQ("Field `title` must be a string.", add_op.error().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 6b0eb75f..2b49a754 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -278,6 +278,79 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(4, results["hits"].size());}+TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {+ Collection *coll1;++ std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");+ std::vector<field> fields = {+ field("max", field_types::INT32, false)+ };++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "max", 0, true).get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ nlohmann::json document = nlohmann::json::parse(json_line);+ Option<nlohmann::json> add_op = coll1->add(document.dump());+ ASSERT_TRUE(add_op.ok());+ }++ infile.close();++ ASSERT_EQ(1, coll1->get_collection_id());+ ASSERT_EQ(3, coll1->get_sort_fields().size());++ std::unordered_map<std::string, field> schema = collection1->get_schema();++ // create a new collection manager to ensure that it restores the records from the disk backed store+ CollectionManager & collectionManager2 = CollectionManager::get_instance();+ collectionManager2.init(store, 1.0, "auth_key");+ auto load_op = collectionManager2.load();++ if(!load_op.ok()) {+ LOG(ERROR) << load_op.error();+ }++ ASSERT_TRUE(load_op.ok());++ auto restored_coll = collectionManager2.get_collection("coll1").get();+ ASSERT_NE(nullptr, restored_coll);++ std::vector<std::string> facet_fields_expected = {};+ auto restored_schema = restored_coll->get_schema();++ ASSERT_EQ(1, restored_coll->get_collection_id());+ ASSERT_EQ(6, restored_coll->get_next_seq_id());+ ASSERT_EQ(facet_fields_expected, restored_coll->get_facet_fields());+ ASSERT_EQ(3, restored_coll->get_sort_fields().size());+ ASSERT_EQ("is_valid", restored_coll->get_sort_fields()[0].name);+ ASSERT_EQ("average", restored_coll->get_sort_fields()[1].name);+ ASSERT_EQ("max", restored_coll->get_sort_fields()[2].name);++ // ensures that the "id" field is not added to the schema+ ASSERT_EQ(6, restored_schema.size());++ ASSERT_EQ("max", restored_coll->get_default_sorting_field());++ ASSERT_EQ(1, restored_schema.count("title"));+ ASSERT_EQ(1, restored_schema.count("max"));+ ASSERT_EQ(1, restored_schema.count("description"));+ ASSERT_EQ(1, restored_schema.count("scores"));+ ASSERT_EQ(1, restored_schema.count("average"));+ ASSERT_EQ(1, restored_schema.count("is_valid"));++ for(const auto& kv: restored_schema) {+ ASSERT_FALSE(kv.second.optional);+ }++ collectionManager.drop_collection("coll1");+ collectionManager2.drop_collection("coll1");+}+TEST_F(CollectionManagerTest, DropCollectionCleanly) {std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::string json_line;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex bc9ecbba..b66c7f47 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1339,7 +1339,7 @@ TEST_F(CollectionTest, ImportDocuments) {"{\"title\": \"Test4\", \"points\": 55, ""\"cast\": [\"Tom Skerritt\"] }"};- import_response = coll_mul_fields->add_many(more_records, document);+ import_response = coll_mul_fields->add_many(more_records, document, CREATE, "", DIRTY_VALUES::REJECT);ASSERT_FALSE(import_response["success"].get<bool>());ASSERT_EQ(2, import_response["num_imported"].get<int>());@@ -1621,7 +1621,19 @@ TEST_F(CollectionTest, IndexingWithBadData) {doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22, \"average\": 78}";const Option<nlohmann::json> & bad_facet_field_op = sample_collection->add(doc_str);ASSERT_FALSE(bad_facet_field_op.ok());- ASSERT_STREQ("Field `tags` must be a string array.", bad_facet_field_op.error().c_str());+ ASSERT_STREQ("Field `tags` must be an array.", bad_facet_field_op.error().c_str());++ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";+ const Option<nlohmann::json> & bad_array_field_op = sample_collection->add(doc_str);+ ASSERT_FALSE(bad_array_field_op.ok());+ ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_op.error().c_str());++ // with coercion should work+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", 22], \"average\": 78}";+ const Option<nlohmann::json> &bad_array_field_coercion_op = sample_collection->add(doc_str, CREATE, "",+ DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(bad_array_field_coercion_op.ok());+ ASSERT_STREQ("Field `tags` must be an array of string.", bad_array_field_coercion_op.error().c_str());doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";const Option<nlohmann::json> & empty_facet_field_op = sample_collection->add(doc_str);@@ -1639,7 +1651,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {bad_default_sorting_field_op3.error().c_str());doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";- const Option<nlohmann::json> & bad_rank_field_op = sample_collection->add(doc_str);+ const Option<nlohmann::json> & bad_rank_field_op = sample_collection->add(doc_str, CREATE, "", DIRTY_VALUES::REJECT);ASSERT_FALSE(bad_rank_field_op.ok());ASSERT_STREQ("Field `average` must be an int32.", bad_rank_field_op.error().c_str());@@ -2007,7 +2019,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {auto add_op = coll1->add(doc.dump());ASSERT_FALSE(add_op.ok());- ASSERT_STREQ("Field `categories` must be a string array.", add_op.error().c_str());+ ASSERT_STREQ("Field `categories` must be an array.", add_op.error().c_str());collectionManager.drop_collection("coll1");}