Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
#include "validator.h"
#include "field.h"
Option<uint32_t> validator_t::coerce_element(const field& a_field, nlohmann::json& document,
nlohmann::json& doc_ele,
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values) {
const std::string& field_name = a_field.name;
bool array_ele_erased = false;
nlohmann::json::iterator dummy_iter;
if(a_field.type == field_types::STRING) {
if(!doc_ele.is_string()) {
Option<uint32_t> coerce_op = coerce_string(dirty_values, fallback_field_type, a_field, document,
field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.type == field_types::INT32) {
if(!doc_ele.is_number_integer()) {
Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.type == field_types::INT64) {
if(!doc_ele.is_number_integer()) {
Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.type == field_types::FLOAT) {
if(!doc_ele.is_number()) {
// using `is_number` allows integer to be passed to a float field
Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.type == field_types::BOOL) {
if(!doc_ele.is_boolean()) {
Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.type == field_types::GEOPOINT) {
if(!doc_ele.is_array() || doc_ele.size() != 2) {
return Option<>(400, "Field `" + field_name + "` must be a 2 element array: [lat, lng].");
}
if(!(doc_ele[0].is_number() && doc_ele[1].is_number())) {
// one or more elements is not a number, try to coerce
Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
doc_ele[0], doc_ele[1],
dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
} else if(a_field.is_array()) {
if (doc_ele.is_null()) {
doc_ele = nlohmann::json::array();
}
if(!doc_ele.is_array()) {
bool is_auto_embedding = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;
if((a_field.optional && (dirty_values == DIRTY_VALUES::DROP ||
dirty_values == DIRTY_VALUES::COERCE_OR_DROP)) || is_auto_embedding) {
document.erase(field_name);
return Option<uint32_t>(200);
} else {
return Option<>(400, "Field `" + field_name + "` must be an array.");
}
}
nlohmann::json::iterator it = doc_ele.begin();
// have to differentiate the geopoint[] type of a nested array object's geopoint[] vs a simple nested field
// geopoint[] type of an array of objects field won't be an array of array
if(a_field.nested && a_field.type == field_types::GEOPOINT_ARRAY && it != doc_ele.end() && it->is_number()) {
if(!doc_ele.empty() && doc_ele.size() % 2 != 0) {
return Option<>(400, "Nested field `" + field_name + "` does not contain valid geopoint values.");
}
const auto& item = doc_ele;
for(size_t ai = 0; ai < doc_ele.size(); ai+=2) {
if(!(doc_ele[ai].is_number() && doc_ele[ai+1].is_number())) {
// one or more elements is not an number, try to coerce
Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
doc_ele[ai], doc_ele[ai+1],
it, true, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
it++;
}
return Option<uint32_t>(200);
}
if(a_field.type == field_types::FLOAT_ARRAY && a_field.num_dim != 0 && a_field.num_dim != doc_ele.size()) {
return Option<uint32_t>(400, "Field `" + a_field.name + "` must have " +
std::to_string(a_field.num_dim) + " dimensions.");
}
for(; it != doc_ele.end(); ) {
nlohmann::json& item = it.value();
array_ele_erased = false;
if (a_field.type == field_types::STRING_ARRAY && !item.is_string()) {
Option<uint32_t> coerce_op = coerce_string(dirty_values, fallback_field_type, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::INT32_ARRAY && !item.is_number_integer()) {
Option<uint32_t> coerce_op = coerce_int32_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::INT64_ARRAY && !item.is_number_integer()) {
Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::FLOAT_ARRAY && !item.is_number()) {
// we check for `is_number` to allow whole numbers to be passed into float fields
Option<uint32_t> coerce_op = coerce_float(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::BOOL_ARRAY && !item.is_boolean()) {
Option<uint32_t> coerce_op = coerce_bool(dirty_values, a_field, document, field_name, it, true, array_ele_erased);
if (!coerce_op.ok()) {
return coerce_op;
}
} else if (a_field.type == field_types::GEOPOINT_ARRAY) {
if(!item.is_array() || item.size() != 2) {
return Option<>(400, "Field `" + field_name + "` must contain 2 element arrays: [ [lat, lng],... ].");
}
if(!(item[0].is_number() && item[1].is_number())) {
// one or more elements is not a number, try to coerce
Option<uint32_t> coerce_op = coerce_geopoint(dirty_values, a_field, document, field_name,
item[0], item[1],
it, true, array_ele_erased);
if(!coerce_op.ok()) {
return coerce_op;
}
}
}
if(!array_ele_erased) {
// if it is erased, the iterator will be reassigned
it++;
}
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::coerce_string(const DIRTY_VALUES& dirty_values, const std::string& fallback_field_type,
const field& a_field, nlohmann::json &document,
const std::string &field_name, nlohmann::json::iterator& array_iter,
bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "an array of" : "a";
auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// we will try to coerce the value to a string
if (item.is_number_integer()) {
item = std::to_string((int64_t)item);
}
else if(item.is_number_float()) {
item = StringUtils::float_to_str((float)item);
}
else if(item.is_boolean()) {
item = item == true ? "true" : "false";
}
else {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " string.");
}
}
return Option<>(200);
}
Option<uint32_t> validator_t::coerce_int32_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "an array of" : "an";
auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// try to value coerce into an integer
if(item.is_number_float()) {
item = static_cast<int32_t>(item.get<float>());
}
else if(item.is_boolean()) {
item = item == true ? 1 : 0;
}
else if(item.is_string() && StringUtils::is_int32_t(item)) {
item = std::atol(item.get<std::string>().c_str());
}
else {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int32.");
}
}
if(document.contains(field_name) && document[field_name].get<int64_t>() > INT32_MAX) {
if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {
document.erase(field_name);
} else {
return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::coerce_int64_t(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "an array of" : "an";
auto& item = is_array ? array_iter.value() : document[field_name];
// Object array reference helper field. It's not provided by the user.
if(is_array && a_field.nested && a_field.is_reference_helper) {
// It's an array of two uint32_t values indicating the object index and referenced doc id respectively.
if(item.size() != 2 || !item.at(0).is_number_unsigned() || !item.at(1).is_number_unsigned()) {
return Option<>(400, "`" + field_name + "` object array reference helper field has wrong value `"
+ item.dump() + "`.");
}
return Option<uint32_t>(200);
}
if(dirty_values == DIRTY_VALUES::REJECT) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// try to value coerce into an integer
if(item.is_number_float()) {
item = static_cast<int64_t>(item.get<float>());
}
else if(item.is_boolean()) {
item = item == true ? 1 : 0;
}
else if(item.is_string() && StringUtils::is_int64_t(item)) {
item = std::atoll(item.get<std::string>().c_str());
}
else {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " int64.");
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::coerce_bool(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "a array of" : "a";
auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// try to value coerce into a bool
if (item.is_number_integer() &&
(item.get<int64_t>() == 1 || item.get<int64_t>() == 0)) {
item = item.get<int64_t>() == 1;
}
else if(item.is_string()) {
std::string str_val = item.get<std::string>();
StringUtils::tolowercase(str_val);
if(str_val == "true") {
item = true;
return Option<uint32_t>(200);
} else if(str_val == "false") {
item = false;
return Option<uint32_t>(200);
} else {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
}
else {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " bool.");
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::coerce_geopoint(const DIRTY_VALUES& dirty_values, const field& a_field,
nlohmann::json &document, const std::string &field_name,
nlohmann::json& lat, nlohmann::json& lng,
nlohmann::json::iterator& array_iter,
bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "an array of" : "a";
if(dirty_values == DIRTY_VALUES::REJECT) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// try to value coerce into a geopoint
if(!lat.is_number() && lat.is_string()) {
if(StringUtils::is_float(lat)) {
lat = std::stof(lat.get<std::string>());
}
}
if(!lng.is_number() && lng.is_string()) {
if(StringUtils::is_float(lng)) {
lng = std::stof(lng.get<std::string>());
}
}
if(!lat.is_number() || !lng.is_number()) {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " geopoint.");
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::coerce_float(const DIRTY_VALUES& dirty_values, const field& a_field, nlohmann::json &document,
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased) {
std::string suffix = is_array ? "a array of" : "a";
auto& item = is_array ? array_iter.value() : document[field_name];
if(dirty_values == DIRTY_VALUES::REJECT) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
if(dirty_values == DIRTY_VALUES::DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
return Option<uint32_t>(200);
}
// try to value coerce into a float
if(item.is_string() && StringUtils::is_float(item)) {
item = std::atof(item.get<std::string>().c_str());
}
else if(item.is_boolean()) {
item = item == true ? 1.0 : 0.0;
}
else {
if(dirty_values == DIRTY_VALUES::COERCE_OR_DROP) {
if(!a_field.optional) {
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
if(!is_array) {
document.erase(field_name);
} else {
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
} else {
// COERCE_OR_REJECT / non-optional + DROP
if(a_field.nested && item.is_array()) {
return Option<>(400, "Field `" + field_name + "` has an incorrect type. "
"Hint: field inside an array of objects must be an array type as well.");
}
return Option<>(400, "Field `" + field_name + "` must be " + suffix + " float.");
}
}
return Option<uint32_t>(200);
}
Option<uint32_t> validator_t::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,
const std::string & default_sorting_field,
const tsl::htrie_map<char, field> & search_schema,
const tsl::htrie_map<char, field> & embedding_fields,
const index_operation_t op,
const bool is_update,
const std::string& fallback_field_type,
const DIRTY_VALUES& dirty_values, const bool validate_embedding_fields) {
bool missing_default_sort_field = (!default_sorting_field.empty() && document.count(default_sorting_field) == 0);
if((op == CREATE || op == UPSERT) && missing_default_sort_field) {
return Option<>(400, "Field `" + default_sorting_field + "` has been declared as a default sorting field, "
"but is not found in the document.");
}
for(const auto& a_field: search_schema) {
const std::string& field_name = a_field.name;
// ignore embedding fields, they will be validated later
if(embedding_fields.count(field_name) > 0) {
continue;
}
if(field_name == "id" || a_field.is_object()) {
continue;
}
if((a_field.optional || op == UPDATE || (op == EMPLACE && is_update)) && document.count(field_name) == 0) {
continue;
}
bool is_auto_embedding = a_field.type == field_types::FLOAT_ARRAY && a_field.embed.count(fields::from) > 0;
if(document.count(field_name) == 0 && !is_auto_embedding && a_field.store) {
return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
"but is not found in the document.");
}
nlohmann::json& doc_ele = document[field_name];
if(a_field.optional && doc_ele.is_null()) {
// we will ignore `null` on an option field
if(!is_update) {
// for updates, the erasure is done later since we need to keep the key for overwrite
document.erase(field_name);
}
continue;
}
auto coerce_op = coerce_element(a_field, document, doc_ele, fallback_field_type, dirty_values);
if(!coerce_op.ok()) {
return coerce_op;
}
}
if(validate_embedding_fields) {
// validate embedding fields
auto validate_embed_op = validate_embed_fields(document, embedding_fields, search_schema, is_update);
if(!validate_embed_op.ok()) {
return Option<>(validate_embed_op.code(), validate_embed_op.error());
}
}
return Option<>(200);
}
Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema,
const bool& is_update) {
for(const auto& field : embedding_fields) {
if(document.contains(field.name) && !is_update) {
const auto& field_vec = document[field.name];
if(!field_vec.is_array() || field_vec.empty() || !field_vec[0].is_number() ||
field_vec.size() != field.num_dim) {
return Option<bool>(400, "Field `" + field.name + "` contains an invalid embedding.");
}
auto it = field_vec.begin();
while(it != field_vec.end()) {
if(!it.value().is_number()) {
return Option<bool>(400, "Field `" + field.name + "` contains invalid float values.");
}
it++;
}
continue;
}
const auto& embed_from = field.embed[fields::from].get<std::vector<std::string>>();
// flag to check if all fields to embed from are optional and null
bool all_optional_and_null = true;
for(const auto& field_name : embed_from) {
auto schema_field_it = search_schema.find(field_name);
auto doc_field_it = document.find(field_name);
if(schema_field_it == search_schema.end()) {
return Option<bool>(400, "Field `" + field.name + "` has invalid fields to create embeddings from.");
}
if(doc_field_it == document.end() || doc_field_it.value().is_null()) {
if(!is_update && !schema_field_it->optional) {
return Option<bool>(400, "Field `" + field_name + "` is needed to create embedding.");
} else {
continue;
}
}
if(doc_field_it.value().is_null()) {
continue;
}
all_optional_and_null = false;
if((schema_field_it.value().type == field_types::STRING && !doc_field_it.value().is_string()) ||
(schema_field_it.value().type == field_types::STRING_ARRAY && !doc_field_it.value().is_array())) {
return Option<bool>(400, "Field `" + field_name + "` has malformed data.");
}
if(doc_field_it.value().is_array()) {
for(const auto& val : doc_field_it.value()) {
if(!val.is_string()) {
return Option<bool>(400, "Field `" + field_name + "` has malformed data.");
}
}
}
}
if(all_optional_and_null && !field.optional && !is_update) {
return Option<bool>(400, "No valid fields found to create embedding for `" + field.name + "`, please provide at least one valid field or make the embedding field optional.");
}
}
return Option<bool>(true);
}