Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle optional restriction for nested fields.
Kishore Nallan869d51568804
diff --git a/include/collection.h b/include/collection.hindex d89ff621..4f4ac1ae 100644--- a/include/collection.h+++ b/include/collection.h@@ -232,7 +232,8 @@ private:static Option<bool> extract_field_name(const std::string& field_name,const tsl::htrie_map<char, field>& search_schema,- std::vector<std::string>& processed_search_fields);+ std::vector<std::string>& processed_search_fields,+ bool extract_only_string_fields);static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,const tsl::htrie_map<char, field>& schema,@@ -245,6 +246,15 @@ private:const nlohmann::json& fdoc, nlohmann::json& fobj,std::vector<std::string>& path_parts, size_t path_index, T func);+ static Option<bool> resolve_field_type(field& new_field,+ nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,+ nlohmann::json& document,+ const DIRTY_VALUES& dirty_values,+ const bool found_dynamic_field,+ const std::string& fallback_field_type,+ std::vector<field>& new_fields,+ std::vector<field>& nested_fields_found);+public:enum {MAX_ARRAY_MATCHES = 5};diff --git a/src/collection.cpp b/src/collection.cppindex 83f3b735..92c1d46a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -734,26 +734,33 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<Option<bool> Collection::extract_field_name(const std::string& field_name,const tsl::htrie_map<char, field>& search_schema,- std::vector<std::string>& processed_search_fields) {- if(search_schema.count(field_name) == 0) {- // we should check if the field refers to an object, which won't be explicitly be present in the schema- auto prefix_it = search_schema.equal_prefix_range(field_name);- bool object_field_found = false;- for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {- // field_name prefix must be followed by a "." to indicate an object search- if(kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.' &&- kv.value().is_string()) {- processed_search_fields.push_back(kv.key());- object_field_found = true;+ std::vector<std::string>& processed_search_fields,+ const bool extract_only_string_fields) {+ auto prefix_it = search_schema.equal_prefix_range(field_name);+ bool field_found = false;++ for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {+ if(extract_only_string_fields && !kv.value().is_string()) {+ if(kv.key().size() == field_name.size() && !kv.value().is_object()) {+ // exact key, must be rejected because of type mismatch+ std::string error = "Field `" + field_name + "` should be a string or a string array.";;+ return Option<bool>(400, error);}+ continue;}- if(!object_field_found) {- std::string error = "Could not find a field named `" + field_name + "` in the schema.";- return Option<bool>(404, error);+ bool exact_non_obj_match = (kv.key().size() == field_name.size() && !kv.value().is_object());++ // field_name prefix must be followed by a "." to indicate an object search+ if(exact_non_obj_match || (kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.')) {+ processed_search_fields.push_back(kv.key());+ field_found = true;}- } else {- processed_search_fields.push_back(field_name);+ }++ if(!field_found) {+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";+ return Option<bool>(404, error);}return Option<bool>(true);@@ -849,9 +856,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,std::vector<std::string> processed_search_fields;for(const std::string& field_name: raw_search_fields) {- auto field_op = extract_field_name(field_name, search_schema, processed_search_fields);+ auto field_op = extract_field_name(field_name, search_schema, processed_search_fields, true);if(!field_op.ok()) {- return Option<nlohmann::json>(404, field_op.error());+ return Option<nlohmann::json>(field_op.code(), field_op.error());}}@@ -873,7 +880,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,std::vector<std::string> group_by_fields;for(const std::string& field_name: raw_group_by_fields) {- auto field_op = extract_field_name(field_name, search_schema, group_by_fields);+ auto field_op = extract_field_name(field_name, search_schema, group_by_fields, false);if(!field_op.ok()) {return Option<nlohmann::json>(404, field_op.error());}@@ -900,14 +907,14 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,tsl::htrie_set<char> exclude_fields_full;for(auto& f_name: include_fields) {- auto field_op = extract_field_name(f_name, search_schema, include_fields_vec);+ auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false);if(!field_op.ok()) {return Option<nlohmann::json>(404, field_op.error());}}for(auto& f_name: exclude_fields) {- auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec);+ auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false);if(!field_op.ok()) {return Option<nlohmann::json>(404, field_op.error());}@@ -1795,7 +1802,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchspp::sparse_hash_set<std::string> fields_highlighted_fully_set;std::vector<std::string> fields_highlighted_fully_expanded;for(const std::string& highlight_full_field: highlight_full_field_names) {- extract_field_name(highlight_full_field, search_schema, fields_highlighted_fully_expanded);+ extract_field_name(highlight_full_field, search_schema, fields_highlighted_fully_expanded, true);}for(std::string & highlight_full_field: fields_highlighted_fully_expanded) {@@ -1834,7 +1841,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search} else {std::vector<std::string> highlight_field_names_expanded;for(size_t i = 0; i < highlight_field_names.size(); i++) {- extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded);+ extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, true);}for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {@@ -3584,14 +3591,14 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,return Option<bool>(true);}-Option<bool> resolve_field_type(field& new_field,- nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,- nlohmann::json& document,- const DIRTY_VALUES& dirty_values,- const bool found_dynamic_field,- const std::string& fallback_field_type,- std::vector<field>& new_fields,- std::vector<field>& nested_fields_found) {+Option<bool> Collection::resolve_field_type(field& new_field,+ nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,+ nlohmann::json& document,+ const DIRTY_VALUES& dirty_values,+ const bool found_dynamic_field,+ const std::string& fallback_field_type,+ std::vector<field>& new_fields,+ std::vector<field>& nested_fields_found) {if(!new_field.index) {return Option<bool>(true);}@@ -3777,19 +3784,15 @@ Index* Collection::init_index() {continue;}- else if(field.nested) {- nested_fields.emplace(field.name, field);- if(!field.is_object()) {- search_schema.emplace(field.name, field);- }- continue;- }-if(field.name == ".*") {continue;}search_schema.emplace(field.name, field);++ if(field.nested) {+ nested_fields.emplace(field.name, field);+ }}synonym_index = new SynonymIndex(store);diff --git a/src/field.cpp b/src/field.cppindex 6909cf9c..10fb1903 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -662,7 +662,7 @@ Option<bool> field::flatten_doc(nlohmann::json& document,StringUtils::split(nested_field.name, field_parts, ".");bool resolved = flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields);- if(!resolved) {+ if(!resolved && !nested_field.optional) {return Option<bool>(400, "Field `" + nested_field.name + "` was not found or has an incorrect type.");}}diff --git a/src/index.cpp b/src/index.cppindex c157a948..4cccf134 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -302,6 +302,82 @@ void Index::compute_token_offsets_facets(index_record& record,}}+bool doc_contains_field(const nlohmann::json& doc, const field& a_field,+ const tsl::htrie_map<char, field> & search_schema) {++ if(doc.count(a_field.name)) {+ return true;+ }++ // check for a nested field, e.g. `foo.bar.baz` indexed but `foo.bar` present in schema+ if(a_field.is_object()) {+ auto prefix_it = search_schema.equal_prefix_range(a_field.name);+ std::string nested_field_name;+ for(auto kv = prefix_it.first; kv != prefix_it.second; kv++) {+ kv.key(nested_field_name);+ bool is_child_field = (nested_field_name.size() > a_field.name.size() &&+ nested_field_name[a_field.name.size()] == '.');+ if(is_child_field && doc.count(nested_field_name) != 0) {+ return true;+ }+ }+ }++ return false;+}++bool validate_object_field(nlohmann::json& doc, const field& a_field) {+ auto field_it = doc.find(a_field.name);+ if(field_it != doc.end()) {+ if(a_field.type == field_types::OBJECT && doc[a_field.name].is_object()) {+ return true;+ } else if(a_field.type == field_types::OBJECT_ARRAY && doc[a_field.name].is_array()) {+ return true;+ }++ return false;+ }++ std::vector<std::string> field_parts;+ StringUtils::split(a_field.name, field_parts, ".");++ nlohmann::json* obj = &doc;+ bool has_array = false;++ for(auto& field_part: field_parts) {+ if(obj->is_array()) {+ has_array = true;++ if(obj->empty()) {+ return false;+ }++ obj = &obj->at(0);+ if(!obj->is_object()) {+ return false;+ }+ }++ auto obj_it = obj->find(field_part);+ if(obj_it == obj->end()) {+ return false;+ }++ obj = &obj_it.value();+ }++ LOG(INFO) << "obj: " << *obj;+ LOG(INFO) << "doc: " << doc;++ if(a_field.type == field_types::OBJECT && obj->is_object()) {+ return true;+ } else if(a_field.type == field_types::OBJECT_ARRAY && (obj->is_array() || (has_array && obj->is_object()))) {+ return true;+ }++ return false;+}+Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,const std::string & default_sorting_field,const tsl::htrie_map<char, field> & search_schema,@@ -319,7 +395,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3for(const auto& a_field: search_schema) {const std::string& field_name = a_field.name;- if(field_name == "id") {+ if(field_name == "id" || a_field.is_object()) {continue;}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 34391db4..daa19ec4 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -886,6 +886,78 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {})"_json;ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());++ // non-optional object field validation (details)+ auto doc2 = R"({+ "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],+ "company": {"name": "Quick and easy fix."},+ "locations": [+ {+ "address": { "street": "Foo bar street" }+ }+ ]+ })"_json;++ add_op = coll3->add(doc2.dump(), CREATE);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `details` was not found or has an incorrect type.", add_op.error());++ // check fields and their properties+ auto coll_fields = coll1->get_fields();+ ASSERT_EQ(6, coll_fields.size());++ for(size_t i = 0; i < coll_fields.size(); i++) {+ auto& coll_field = coll_fields[i];+ if(i <= 2) {+ // original 3 explicit fields will be non-optional, but the sub-properties will be optional+ ASSERT_FALSE(coll_field.optional);+ } else {+ ASSERT_TRUE(coll_field.optional);+ }+ }+}++TEST_F(CollectionNestedFieldsTest, ExplicitSchemaOptionalFieldValidation) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "details", "type": "object", "optional": true },+ {"name": "company.name", "type": "string", "optional": true },+ {"name": "locations", "type": "object[]", "optional": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ // no optional field is present and that should be allowed+ auto doc1 = R"({+ "foo": "bar"+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ // some parts of an optional field is present in a subsequent doc indexed+ auto doc2 = R"({+ "details": {"name": "foo"}+ })"_json;+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto doc3 = R"({+ "details": {"age": 30}+ })"_json;+ add_op = coll1->add(doc3.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ // check fields and their properties+ auto coll_fields = coll1->get_fields();+ ASSERT_EQ(5, coll_fields.size());+ for(auto& coll_field : coll_fields) {+ ASSERT_TRUE(coll_field.optional);+ }}TEST_F(CollectionNestedFieldsTest, SortByNestedField) {