Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle optional restriction for nested fields.

Kishore Nallan869d51568804
diff --git a/include/collection.h b/include/collection.h
index d89ff621..4f4ac1ae 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -232,7 +232,8 @@ private:
static Option<bool> extract_field_name(const std::string& field_name,
const tsl::htrie_map<char, field>& search_schema,
- std::vector<std::string>& processed_search_fields);
+ std::vector<std::string>& processed_search_fields,
+ bool extract_only_string_fields);
static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,
const tsl::htrie_map<char, field>& schema,
@@ -245,6 +246,15 @@ private:
const nlohmann::json& fdoc, nlohmann::json& fobj,
std::vector<std::string>& path_parts, size_t path_index, T func);
+ static Option<bool> resolve_field_type(field& new_field,
+ nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
+ nlohmann::json& document,
+ const DIRTY_VALUES& dirty_values,
+ const bool found_dynamic_field,
+ const std::string& fallback_field_type,
+ std::vector<field>& new_fields,
+ std::vector<field>& nested_fields_found);
+
public:
enum {MAX_ARRAY_MATCHES = 5};
diff --git a/src/collection.cpp b/src/collection.cpp
index 83f3b735..92c1d46a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -734,26 +734,33 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
Option<bool> Collection::extract_field_name(const std::string& field_name,
const tsl::htrie_map<char, field>& search_schema,
- std::vector<std::string>& processed_search_fields) {
- if(search_schema.count(field_name) == 0) {
- // we should check if the field refers to an object, which won't be explicitly be present in the schema
- auto prefix_it = search_schema.equal_prefix_range(field_name);
- bool object_field_found = false;
- for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {
- // field_name prefix must be followed by a "." to indicate an object search
- if(kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.' &&
- kv.value().is_string()) {
- processed_search_fields.push_back(kv.key());
- object_field_found = true;
+ std::vector<std::string>& processed_search_fields,
+ const bool extract_only_string_fields) {
+ auto prefix_it = search_schema.equal_prefix_range(field_name);
+ bool field_found = false;
+
+ for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {
+ if(extract_only_string_fields && !kv.value().is_string()) {
+ if(kv.key().size() == field_name.size() && !kv.value().is_object()) {
+ // exact key, must be rejected because of type mismatch
+ std::string error = "Field `" + field_name + "` should be a string or a string array.";;
+ return Option<bool>(400, error);
}
+ continue;
}
- if(!object_field_found) {
- std::string error = "Could not find a field named `" + field_name + "` in the schema.";
- return Option<bool>(404, error);
+ bool exact_non_obj_match = (kv.key().size() == field_name.size() && !kv.value().is_object());
+
+ // field_name prefix must be followed by a "." to indicate an object search
+ if(exact_non_obj_match || (kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.')) {
+ processed_search_fields.push_back(kv.key());
+ field_found = true;
}
- } else {
- processed_search_fields.push_back(field_name);
+ }
+
+ if(!field_found) {
+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";
+ return Option<bool>(404, error);
}
return Option<bool>(true);
@@ -849,9 +856,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
std::vector<std::string> processed_search_fields;
for(const std::string& field_name: raw_search_fields) {
- auto field_op = extract_field_name(field_name, search_schema, processed_search_fields);
+ auto field_op = extract_field_name(field_name, search_schema, processed_search_fields, true);
if(!field_op.ok()) {
- return Option<nlohmann::json>(404, field_op.error());
+ return Option<nlohmann::json>(field_op.code(), field_op.error());
}
}
@@ -873,7 +880,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
std::vector<std::string> group_by_fields;
for(const std::string& field_name: raw_group_by_fields) {
- auto field_op = extract_field_name(field_name, search_schema, group_by_fields);
+ auto field_op = extract_field_name(field_name, search_schema, group_by_fields, false);
if(!field_op.ok()) {
return Option<nlohmann::json>(404, field_op.error());
}
@@ -900,14 +907,14 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
tsl::htrie_set<char> exclude_fields_full;
for(auto& f_name: include_fields) {
- auto field_op = extract_field_name(f_name, search_schema, include_fields_vec);
+ auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false);
if(!field_op.ok()) {
return Option<nlohmann::json>(404, field_op.error());
}
}
for(auto& f_name: exclude_fields) {
- auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec);
+ auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false);
if(!field_op.ok()) {
return Option<nlohmann::json>(404, field_op.error());
}
@@ -1795,7 +1802,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
spp::sparse_hash_set<std::string> fields_highlighted_fully_set;
std::vector<std::string> fields_highlighted_fully_expanded;
for(const std::string& highlight_full_field: highlight_full_field_names) {
- extract_field_name(highlight_full_field, search_schema, fields_highlighted_fully_expanded);
+ extract_field_name(highlight_full_field, search_schema, fields_highlighted_fully_expanded, true);
}
for(std::string & highlight_full_field: fields_highlighted_fully_expanded) {
@@ -1834,7 +1841,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
} else {
std::vector<std::string> highlight_field_names_expanded;
for(size_t i = 0; i < highlight_field_names.size(); i++) {
- extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded);
+ extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, true);
}
for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {
@@ -3584,14 +3591,14 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
return Option<bool>(true);
}
-Option<bool> resolve_field_type(field& new_field,
- nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
- nlohmann::json& document,
- const DIRTY_VALUES& dirty_values,
- const bool found_dynamic_field,
- const std::string& fallback_field_type,
- std::vector<field>& new_fields,
- std::vector<field>& nested_fields_found) {
+Option<bool> Collection::resolve_field_type(field& new_field,
+ nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
+ nlohmann::json& document,
+ const DIRTY_VALUES& dirty_values,
+ const bool found_dynamic_field,
+ const std::string& fallback_field_type,
+ std::vector<field>& new_fields,
+ std::vector<field>& nested_fields_found) {
if(!new_field.index) {
return Option<bool>(true);
}
@@ -3777,19 +3784,15 @@ Index* Collection::init_index() {
continue;
}
- else if(field.nested) {
- nested_fields.emplace(field.name, field);
- if(!field.is_object()) {
- search_schema.emplace(field.name, field);
- }
- continue;
- }
-
if(field.name == ".*") {
continue;
}
search_schema.emplace(field.name, field);
+
+ if(field.nested) {
+ nested_fields.emplace(field.name, field);
+ }
}
synonym_index = new SynonymIndex(store);
diff --git a/src/field.cpp b/src/field.cpp
index 6909cf9c..10fb1903 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -662,7 +662,7 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
StringUtils::split(nested_field.name, field_parts, ".");
bool resolved = flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields);
- if(!resolved) {
+ if(!resolved && !nested_field.optional) {
return Option<bool>(400, "Field `" + nested_field.name + "` was not found or has an incorrect type.");
}
}
diff --git a/src/index.cpp b/src/index.cpp
index c157a948..4cccf134 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -302,6 +302,82 @@ void Index::compute_token_offsets_facets(index_record& record,
}
}
+bool doc_contains_field(const nlohmann::json& doc, const field& a_field,
+ const tsl::htrie_map<char, field> & search_schema) {
+
+ if(doc.count(a_field.name)) {
+ return true;
+ }
+
+ // check for a nested field, e.g. `foo.bar.baz` indexed but `foo.bar` present in schema
+ if(a_field.is_object()) {
+ auto prefix_it = search_schema.equal_prefix_range(a_field.name);
+ std::string nested_field_name;
+ for(auto kv = prefix_it.first; kv != prefix_it.second; kv++) {
+ kv.key(nested_field_name);
+ bool is_child_field = (nested_field_name.size() > a_field.name.size() &&
+ nested_field_name[a_field.name.size()] == '.');
+ if(is_child_field && doc.count(nested_field_name) != 0) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+bool validate_object_field(nlohmann::json& doc, const field& a_field) {
+ auto field_it = doc.find(a_field.name);
+ if(field_it != doc.end()) {
+ if(a_field.type == field_types::OBJECT && doc[a_field.name].is_object()) {
+ return true;
+ } else if(a_field.type == field_types::OBJECT_ARRAY && doc[a_field.name].is_array()) {
+ return true;
+ }
+
+ return false;
+ }
+
+ std::vector<std::string> field_parts;
+ StringUtils::split(a_field.name, field_parts, ".");
+
+ nlohmann::json* obj = &doc;
+ bool has_array = false;
+
+ for(auto& field_part: field_parts) {
+ if(obj->is_array()) {
+ has_array = true;
+
+ if(obj->empty()) {
+ return false;
+ }
+
+ obj = &obj->at(0);
+ if(!obj->is_object()) {
+ return false;
+ }
+ }
+
+ auto obj_it = obj->find(field_part);
+ if(obj_it == obj->end()) {
+ return false;
+ }
+
+ obj = &obj_it.value();
+ }
+
+ LOG(INFO) << "obj: " << *obj;
+ LOG(INFO) << "doc: " << doc;
+
+ if(a_field.type == field_types::OBJECT && obj->is_object()) {
+ return true;
+ } else if(a_field.type == field_types::OBJECT_ARRAY && (obj->is_array() || (has_array && obj->is_object()))) {
+ return true;
+ }
+
+ return false;
+}
+
Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,
const std::string & default_sorting_field,
const tsl::htrie_map<char, field> & search_schema,
@@ -319,7 +395,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
for(const auto& a_field: search_schema) {
const std::string& field_name = a_field.name;
- if(field_name == "id") {
+ if(field_name == "id" || a_field.is_object()) {
continue;
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 34391db4..daa19ec4 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -886,6 +886,78 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
})"_json;
ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+
+ // non-optional object field validation (details)
+ auto doc2 = R"({
+ "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],
+ "company": {"name": "Quick and easy fix."},
+ "locations": [
+ {
+ "address": { "street": "Foo bar street" }
+ }
+ ]
+ })"_json;
+
+ add_op = coll3->add(doc2.dump(), CREATE);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `details` was not found or has an incorrect type.", add_op.error());
+
+ // check fields and their properties
+ auto coll_fields = coll1->get_fields();
+ ASSERT_EQ(6, coll_fields.size());
+
+ for(size_t i = 0; i < coll_fields.size(); i++) {
+ auto& coll_field = coll_fields[i];
+ if(i <= 2) {
+ // original 3 explicit fields will be non-optional, but the sub-properties will be optional
+ ASSERT_FALSE(coll_field.optional);
+ } else {
+ ASSERT_TRUE(coll_field.optional);
+ }
+ }
+}
+
+TEST_F(CollectionNestedFieldsTest, ExplicitSchemaOptionalFieldValidation) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "details", "type": "object", "optional": true },
+ {"name": "company.name", "type": "string", "optional": true },
+ {"name": "locations", "type": "object[]", "optional": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ // no optional field is present and that should be allowed
+ auto doc1 = R"({
+ "foo": "bar"
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ // some parts of an optional field is present in a subsequent doc indexed
+ auto doc2 = R"({
+ "details": {"name": "foo"}
+ })"_json;
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto doc3 = R"({
+ "details": {"age": 30}
+ })"_json;
+ add_op = coll1->add(doc3.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ // check fields and their properties
+ auto coll_fields = coll1->get_fields();
+ ASSERT_EQ(5, coll_fields.size());
+ for(auto& coll_field : coll_fields) {
+ ASSERT_TRUE(coll_field.optional);
+ }
}
TEST_F(CollectionNestedFieldsTest, SortByNestedField) {