Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Multiple explicit nested fields should be allowed.
Kishore Nallana90bfba9305f
diff --git a/include/collection.h b/include/collection.hindex 36116f04..d89ff621 100644--- a/include/collection.h+++ b/include/collection.h@@ -110,6 +110,10 @@ private:std::unordered_map<std::string, field> dynamic_fields;+ tsl::htrie_map<char, field> nested_fields;++ bool nested_fields_enabled;+std::vector<char> symbols_to_index;std::vector<char> token_separators;@@ -158,6 +162,7 @@ private:const DIRTY_VALUES& dirty_values,const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,+ const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,std::vector<field>& new_fields);@@ -229,7 +234,7 @@ private:const tsl::htrie_map<char, field>& search_schema,std::vector<std::string>& processed_search_fields);- static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,+ static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,const tsl::htrie_map<char, field>& schema,std::vector<field>& new_fields);@@ -262,6 +267,7 @@ public:static constexpr const char* COLLECTION_CREATED = "created_at";static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";static constexpr const char* COLLECTION_FALLBACK_FIELD_TYPE = "fallback_field_type";+ static constexpr const char* COLLECTION_NESTED_FIELDS_ENABLED = "nested_fields_enabled";static constexpr const char* COLLECTION_SYMBOLS_TO_INDEX = "symbols_to_index";static constexpr const char* COLLECTION_SEPARATORS = "token_separators";@@ -274,7 +280,8 @@ public:const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,const std::string& default_sorting_field,const float max_memory_ratio, const std::string& fallback_field_type,- const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators);+ const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators,+ const bool nested_fields_enabled);~Collection();diff --git a/include/collection_manager.h b/include/collection_manager.hindex 78feeaa2..618562aa 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -152,7 +152,8 @@ public:const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)),const std::string& fallback_field_type = "",const std::vector<std::string>& symbols_to_index = {},- const std::vector<std::string>& token_separators = {});+ const std::vector<std::string>& token_separators = {},+ const bool nested_fields_enabled = false);locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;diff --git a/include/field.h b/include/field.hindex c32d3b32..b1dde967 100644--- a/include/field.h+++ b/include/field.h@@ -163,18 +163,13 @@ struct field {return !is_array();}- bool is_dynamic() const {- return is_dynamic_type() || nested;- }-- static bool is_dynamic_type(const std::string& name, const std::string& type) {+ static bool is_dynamic(const std::string& name, const std::string& type) {return type == "string*" || (name != ".*" && type == field_types::AUTO) ||- type == field_types::OBJECT || type == field_types::OBJECT_ARRAY ||(name != ".*" && name.find(".*") != std::string::npos);}- bool is_dynamic_type() const {- return is_dynamic_type(name, type);+ bool is_dynamic() const {+ return is_dynamic(name, type);}bool has_numerical_index() const {diff --git a/src/collection.cpp b/src/collection.cppindex 56feff45..4f9844c4 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -23,10 +23,12 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, coconst uint32_t next_seq_id, Store *store, const std::vector<field> &fields,const std::string& default_sorting_field,const float max_memory_ratio, const std::string& fallback_field_type,- const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators):+ const std::vector<std::string>& symbols_to_index,+ const std::vector<std::string>& token_separators,+ const bool nested_fields_enabled):name(name), collection_id(collection_id), created_at(created_at),next_seq_id(next_seq_id), store(store),- fields(fields), default_sorting_field(default_sorting_field),+ fields(fields), default_sorting_field(default_sorting_field), nested_fields_enabled(nested_fields_enabled),max_memory_ratio(max_memory_ratio),fallback_field_type(fallback_field_type), dynamic_fields({}),symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),@@ -229,12 +231,13 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmabatch_doc_ids.insert(doc_id);// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing- if(!fallback_field_type.empty() || !dynamic_fields.empty()) {+ if(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty()) {std::vector<field> new_fields;std::unique_lock lock(mutex);Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,search_schema, dynamic_fields,+ nested_fields,fallback_field_type,new_fields);if(!new_fields_op.ok()) {@@ -3480,6 +3483,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,std::vector<field> new_fields;Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,updated_search_schema, new_dynamic_fields,+ nested_fields,fallback_field_type,new_fields);if(!new_fields_op.ok()) {@@ -3548,13 +3552,104 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,return Option<bool>(true);}+Option<bool> resolve_field_type(field& new_field,+ nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,+ nlohmann::json& document,+ const DIRTY_VALUES& dirty_values,+ const bool found_dynamic_field,+ const std::string& fallback_field_type,+ std::vector<field>& new_fields,+ std::vector<field>& nested_fields_found) {+ if(!new_field.index) {+ return Option<bool>(true);+ }++ // Type detection scenarios:+ // a) Not a dynamic field + fallback type is explicit: use fallback type+ // b) Dynamic field + type is explicit: use explicit type+ // c) Not a dynamic field + fallback type is auto: detect and assign type+ // d) Dynamic field + type is auto: detect and assign type+ // e) Not a dynamic field + fallback type is string*: map to string/string[]+ // f) Dynamic field + type is string*: map to string/string[]++ const std::string& test_field_type = found_dynamic_field ? new_field.type : fallback_field_type;++ if(test_field_type == field_types::AUTO || field_types::is_string_or_array(test_field_type)) {+ if(kv.key() == ".*") {+ return Option<bool>(true);+ }++ std::string field_type;+ bool parseable = field::get_type(kv.value(), field_type);+ if(!parseable) {++ if(kv.value().is_null() && new_field.optional) {+ // null values are allowed only if field is optional+ kv = document.erase(kv);+ return Option<bool>(false);+ }++ if(kv.value().is_object()) {+ return Option<bool>(true);+ }++ if(kv.value().is_array() && kv.value().empty()) {+ return Option<bool>(true);+ }++ if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {+ return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");+ } else {+ // DROP or COERCE_OR_DROP+ kv = document.erase(kv);+ return Option<bool>(false);+ }+ }++ if(test_field_type == field_types::AUTO) {+ new_field.type = field_type;+ if(new_field.is_object()) {+ new_field.nested = true;+ }+ } else {+ if (kv.value().is_array()) {+ new_field.type = field_types::STRING_ARRAY;+ } else {+ new_field.type = field_types::STRING;+ }+ }+ }++ else {+ new_field.type = test_field_type;+ }++ if (new_field.is_num_sort_field()) {+ // only numerical fields are added to sort index in dynamic type detection+ new_field.sort = true;+ }++ if(new_field.nested) {+ nested_fields_found.emplace_back(new_field);+ } else {+ new_fields.emplace_back(new_field);+ }++ return Option<bool>(true);+}+Option<bool> Collection::detect_new_fields(nlohmann::json& document,const DIRTY_VALUES& dirty_values,const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,+ const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,std::vector<field>& new_fields) {- std::vector<field> nested_fields;++ std::vector<field> nested_fields_found;+ for(auto& nested_field: nested_fields) {+ nested_fields_found.push_back(nested_field);+ }auto kv = document.begin();while(kv != document.end()) {@@ -3562,27 +3657,17 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,if (schema.count(kv.key()) == 0 && kv.key() != "id") {const std::string &fname = kv.key();field new_field(fname, field_types::STRING, false, true);- std::string field_type;- bool parseable;-bool found_dynamic_field = false;bool skip_field = false;// check against dynamic field definitions- for(const auto& dynamic_field: dyn_fields) {- if (dynamic_field.second.nested && (kv.key() == dynamic_field.first ||- (StringUtils::begins_with(dynamic_field.first, kv.key()) &&- dynamic_field.first[kv.key().size()] == '.'))) {- new_field = dynamic_field.second;- new_field.name = dynamic_field.first;- found_dynamic_field = true;- break;- }+ for(auto dyn_field_it = dyn_fields.begin(); dyn_field_it != dyn_fields.end(); dyn_field_it++) {+ auto& dynamic_field = dyn_field_it->second;- else if(std::regex_match (kv.key(), std::regex(dynamic_field.first))) {+ if(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {// unless the field is auto or string*, ignore field name matching regexp pattern- if(kv.key() == dynamic_field.first && !dynamic_field.second.is_auto() &&- !dynamic_field.second.is_string_star()) {+ if(kv.key() == dynamic_field.name && !dynamic_field.is_auto() &&+ !dynamic_field.is_string_star()) {skip_field = true;break;}@@ -3593,7 +3678,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,break;}- new_field = dynamic_field.second;+ new_field = dynamic_field;new_field.name = fname;found_dynamic_field = true;break;@@ -3611,101 +3696,34 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,continue;}- if(!new_field.index) {- kv++;- continue;- }-- // Type detection scenarios:- // a) Not a dynamic field + fallback type is explicit: use fallback type- // b) Dynamic field + type is explicit: use explicit type- // c) Not a dynamic field + fallback type is auto: detect and assign type- // d) Dynamic field + type is auto: detect and assign type- // e) Not a dynamic field + fallback type is string*: map to string/string[]- // f) Dynamic field + type is string*: map to string/string[]-- const std::string& test_field_type = found_dynamic_field ? new_field.type : fallback_field_type;-- if(test_field_type == field_types::AUTO || field_types::is_string_or_array(test_field_type)) {- if(kv.key() == ".*") {- kv++;- continue;- }-- parseable = field::get_type(kv.value(), field_type);- if(!parseable) {-- if(kv.value().is_null() && new_field.optional) {- // null values are allowed only if field is optional- kv = document.erase(kv);- continue;- }-- if(kv.value().is_object()) {- kv++;- continue;- }-- if(kv.value().is_array() && kv.value().empty()) {- kv++;- continue;- }-- if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {- return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");- } else {- // DROP or COERCE_OR_DROP- kv = document.erase(kv);- continue;- }- }-- if(test_field_type == field_types::AUTO) {- new_field.type = field_type;- if(new_field.is_object()) {- new_field.nested = true;- }- } else {- if (kv.value().is_array()) {- new_field.type = field_types::STRING_ARRAY;- } else {- new_field.type = field_types::STRING;- }- }-+ auto add_op = resolve_field_type(new_field, kv, document, dirty_values, found_dynamic_field,+ fallback_field_type, new_fields, nested_fields_found);+ if(!add_op.ok()) {+ return add_op;}- else {- new_field.type = test_field_type;- }-- if (new_field.is_num_sort_field()) {- // only numerical fields are added to sort index in dynamic type detection- new_field.sort = true;- }-- if(new_field.nested) {- nested_fields.push_back(new_field);- } else {- new_fields.emplace_back(new_field);+ bool increment_iter = add_op.get();+ if(!increment_iter) {+ continue;}}kv++;}- return flatten_and_identify_new_fields(document, nested_fields, schema, new_fields);+ return flatten_and_identify_new_fields(document, nested_fields_found, schema, new_fields);}-Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,- const tsl::htrie_map<char, field>& schema,- std::vector<field>& new_fields) {- if(nested_fields.empty()) {+Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc,+ const std::vector<field>& nested_fields_found,+ const tsl::htrie_map<char, field>& schema,+ std::vector<field>& new_fields) {+ if(nested_fields_found.empty()) {return Option<bool>(true);}std::vector<field> flattened_fields;- auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, nested_fields_found, flattened_fields);if(!flatten_op.ok()) {return flatten_op;}@@ -3724,7 +3742,12 @@ Index* Collection::init_index() {if(field.is_dynamic()) {// regexp fields and fields with auto type are treated as dynamic fieldsdynamic_fields.emplace(field.name, field);- if(!field.is_dynamic_type()) {+ continue;+ }++ else if(field.nested) {+ nested_fields.emplace(field.name, field);+ if(!field.is_object()) {search_schema.emplace(field.name, field);}continue;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex bb3b7801..10502dfe 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -69,6 +69,10 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectioncollection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE].get<std::string>() :"";+ bool nested_fields_enabled = collection_meta.count(Collection::COLLECTION_NESTED_FIELDS_ENABLED) != 0 ?+ collection_meta[Collection::COLLECTION_NESTED_FIELDS_ENABLED].get<bool>() :+ false;+std::vector<std::string> symbols_to_index;std::vector<std::string> token_separators;@@ -92,7 +96,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionmax_memory_ratio,fallback_field_type,symbols_to_index,- token_separators);+ token_separators,+ nested_fields_enabled);return collection;}@@ -291,7 +296,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameconst uint64_t created_at,const std::string& fallback_field_type,const std::vector<std::string>& symbols_to_index,- const std::vector<std::string>& token_separators) {+ const std::vector<std::string>& token_separators,+ const bool nested_fields_enabled) {std::unique_lock lock(mutex);if(store->contains(Collection::get_meta_key(name))) {@@ -324,11 +330,13 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namecollection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE] = fallback_field_type;collection_meta[Collection::COLLECTION_SYMBOLS_TO_INDEX] = symbols_to_index;collection_meta[Collection::COLLECTION_SEPARATORS] = token_separators;+ collection_meta[Collection::COLLECTION_NESTED_FIELDS_ENABLED] = nested_fields_enabled;Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,default_sorting_field,this->max_memory_ratio, fallback_field_type,- symbols_to_index, token_separators);+ symbols_to_index, token_separators,+ nested_fields_enabled);next_collection_id++;rocksdb::WriteBatch batch;@@ -970,6 +978,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoconst char* NUM_MEMORY_SHARDS = "num_memory_shards";const char* SYMBOLS_TO_INDEX = "symbols_to_index";const char* TOKEN_SEPARATORS = "token_separators";+ const char* NESTED_FIELDS_ENABLED = "nested_fields_enabled";const char* DEFAULT_SORTING_FIELD = "default_sorting_field";// validate presence of mandatory fields@@ -994,6 +1003,10 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoreq_json[TOKEN_SEPARATORS] = std::vector<std::string>();}+ if(req_json.count(NESTED_FIELDS_ENABLED) == 0) {+ req_json[NESTED_FIELDS_ENABLED] = false;+ }+if(req_json.count("fields") == 0) {return Option<Collection*>(400, "Parameter `fields` is required.");}@@ -1019,6 +1032,10 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoreturn Option<Collection*>(400, std::string("`") + TOKEN_SEPARATORS + "` should be an array of character symbols.");}+ if(!req_json[NESTED_FIELDS_ENABLED].is_boolean()) {+ return Option<Collection*>(400, std::string("`") + NESTED_FIELDS_ENABLED + "` should be a boolean.");+ }+for (auto it = req_json[SYMBOLS_TO_INDEX].begin(); it != req_json[SYMBOLS_TO_INDEX].end(); ++it) {if(!it->is_string() || it->get<std::string>().size() != 1 ) {return Option<Collection*>(400, std::string("`") + SYMBOLS_TO_INDEX + "` should be an array of character symbols.");@@ -1059,7 +1076,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsofields, default_sorting_field, created_at,fallback_field_type,req_json[SYMBOLS_TO_INDEX],- req_json[TOKEN_SEPARATORS]);+ req_json[TOKEN_SEPARATORS],+ req_json[NESTED_FIELDS_ENABLED]);}Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,diff --git a/src/field.cpp b/src/field.cppindex a59c5b2c..6909cf9c 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -510,7 +510,7 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<if(field_json.count(fields::optional) == 0) {// dynamic type fields are always optional- bool is_dynamic = field::is_dynamic_type(field_json[fields::name], field_json[fields::type]);+ bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);field_json[fields::optional] = is_dynamic;}diff --git a/src/index.cpp b/src/index.cppindex 3182ca22..fd569589 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4849,7 +4849,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecstd::unique_lock lock(mutex);for(const auto & new_field: new_fields) {- if(!new_field.index || new_field.is_dynamic_type()) {+ if(!new_field.index || new_field.is_dynamic()) {continue;}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex eedfe348..a52bf011 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -99,7 +99,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {"{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"sort\":true,\"type\":\"geopoint\"},""{\"facet\":false,\"index\":false,\"infix\":false,\"locale\":\"\",\"name\":\"not_stored\",\"optional\":true,\"sort\":false,\"type\":\"string\"},""{\"facet\":false,\"index\":true,\"infix\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"sort\":true,\"type\":\"int32\"}],\"id\":0,"- "\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",+ "\"name\":\"collection1\",\"nested_fields_enabled\":false,\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",collection_meta_json);ASSERT_EQ("1", next_collection_id);}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex eddd8666..34391db4 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -768,11 +768,6 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {auto add_op = coll1->add(doc.dump(), CREATE);ASSERT_TRUE(add_op.ok());- auto sop = coll1->search("brown fox", {"details", "locations"},- "", {}, sort_fields, {0}, 10, 1,- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4);-// search both simply nested and deeply nested array-of-objectsauto results = coll1->search("brown fox", {"details", "locations"},"", {}, sort_fields, {0}, 10, 1,@@ -803,12 +798,6 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());- sop = coll1->search("fix", {"company.name"},- "", {}, sort_fields, {0}, 10, 1,- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4);--results = coll1->search("fix", {"company.name"},"", {}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),@@ -987,6 +976,28 @@ TEST_F(CollectionNestedFieldsTest, SortByNestedField) {ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());}+TEST_F(CollectionNestedFieldsTest, OnlyExplcitSchemaFieldMustBeIndexedInADoc) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "company.num_employees", "type": "int32", "optional": false },+ {"name": "company.founded", "type": "int32", "optional": false }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "company": {"num_employees": 2000, "founded": 1976, "year": 2000}+ })"_json;++ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());+ auto fs = coll1->get_fields();+ ASSERT_EQ(2, coll1->get_fields().size());+}+TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {std::vector<field> fields = {field(".*", field_types::AUTO, false, true),field("education.name", field_types::STRING_ARRAY, true, true),