Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support nested dynamic fields.
Kishore Nallan806008f42143
diff --git a/include/field.h b/include/field.hindex d4361db9..4fa6cc6e 100644--- a/include/field.h+++ b/include/field.h@@ -489,13 +489,17 @@ struct field {static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,const field& the_field, const std::string& flat_name,+ const std::unordered_map<std::string, field>& dyn_fields,std::unordered_map<std::string, field>& flattened_fields);static Option<bool> flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,std::vector<std::string>& path_parts, size_t path_index, bool has_array,- bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);+ bool has_obj_array,+ const std::unordered_map<std::string, field>& dyn_fields,+ std::unordered_map<std::string, field>& flattened_fields);static Option<bool> flatten_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields,+ const std::unordered_map<std::string, field>& dyn_fields,bool missing_is_ok, std::vector<field>& flattened_fields);static void compact_nested_fields(tsl::htrie_map<char, field>& nested_fields);diff --git a/src/collection.cpp b/src/collection.cppindex e4737326..90ca8b1a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3511,7 +3511,7 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,if(!raw_doc && enable_nested_fields) {std::vector<field> flattened_fields;- field::flatten_doc(document, nested_fields, true, flattened_fields);+ field::flatten_doc(document, nested_fields, {}, true, flattened_fields);}return Option<bool>(true);@@ -3700,7 +3700,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsif(enable_nested_fields) {std::vector<field> flattened_fields;- field::flatten_doc(document, nested_fields, true, flattened_fields);+ field::flatten_doc(document, nested_fields, {}, true, flattened_fields);}index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);@@ -4383,13 +4383,6 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,auto& dynamic_field = dyn_field_it->second;if(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {- // unless the field is auto or string*, ignore field name matching regexp pattern- if(kv.key() == dynamic_field.name && !dynamic_field.is_auto() &&- !dynamic_field.is_string_star()) {- skip_field = true;- break;- }-// to prevent confusion we also disallow dynamic field names that contain ".*"if((kv.key() != ".*" && kv.key().find(".*") != std::string::npos)) {skip_field = true;@@ -4437,7 +4430,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,}std::vector<field> flattened_fields;- auto flatten_op = field::flatten_doc(document, nested_fields, is_update, flattened_fields);+ auto flatten_op = field::flatten_doc(document, nested_fields, dyn_fields, is_update, flattened_fields);if(!flatten_op.ok()) {return flatten_op;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 28bd0f5a..52b6ce15 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1322,7 +1322,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionif(collection->get_enable_nested_fields()) {std::vector<field> flattened_fields;- field::flatten_doc(document, collection->get_nested_fields(), true, flattened_fields);+ field::flatten_doc(document, collection->get_nested_fields(), {}, true, flattened_fields);}auto dirty_values = DIRTY_VALUES::DROP;diff --git a/src/field.cpp b/src/field.cppindex 6c572b99..bcc8470a 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -4,6 +4,7 @@#include "text_embedder_manager.h"#include <stack>#include <collection_manager.h>+#include <regex>Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,const std::string& format_err_msg,@@ -802,15 +803,17 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsobool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,const field& the_field, const std::string& flat_name,+ const std::unordered_map<std::string, field>& dyn_fields,std::unordered_map<std::string, field>& flattened_fields) {if(value.is_object()) {has_obj_array = has_array;for(const auto& kv: value.items()) {- flatten_obj(doc, kv.value(), has_array, has_obj_array, the_field, flat_name + "." + kv.key(), flattened_fields);+ flatten_obj(doc, kv.value(), has_array, has_obj_array, the_field, flat_name + "." + kv.key(),+ dyn_fields, flattened_fields);}} else if(value.is_array()) {for(const auto& kv: value.items()) {- flatten_obj(doc, kv.value(), true, has_obj_array, the_field, flat_name, flattened_fields);+ flatten_obj(doc, kv.value(), true, has_obj_array, the_field, flat_name, dyn_fields, flattened_fields);}} else { // must be a primitiveif(doc.count(flat_name) != 0 && flattened_fields.find(flat_name) == flattened_fields.end()) {@@ -818,8 +821,31 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arr}std::string detected_type;- if(!field::get_type(value, detected_type)) {- return false;+ bool found_dynamic_field = false;++ for(auto dyn_field_it = dyn_fields.begin(); dyn_field_it != dyn_fields.end(); dyn_field_it++) {+ auto& dynamic_field = dyn_field_it->second;++ if(dynamic_field.is_auto() || dynamic_field.is_string_star()) {+ continue;+ }++ if(std::regex_match(flat_name, std::regex(flat_name))) {+ detected_type = dynamic_field.type;+ found_dynamic_field = true;+ break;+ }+ }++ if(!found_dynamic_field) {+ if(!field::get_type(value, detected_type)) {+ return false;+ }++ if(std::isalnum(detected_type.back()) && has_array) {+ // convert singular type to multi valued type+ detected_type += "[]";+ }}if(has_array) {@@ -828,11 +854,6 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arrdoc[flat_name] = value;}- if(std::isalnum(detected_type.back()) && has_array) {- // convert singular type to multi valued type- detected_type += "[]";- }-field flattened_field = the_field;flattened_field.name = flat_name;flattened_field.type = detected_type;@@ -848,22 +869,42 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arrOption<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,std::vector<std::string>& path_parts, size_t path_index,- bool has_array, bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields) {+ bool has_array, bool has_obj_array,+ const std::unordered_map<std::string, field>& dyn_fields,+ std::unordered_map<std::string, field>& flattened_fields) {if(path_index == path_parts.size()) {// end of path: check if obj matches expected typestd::string detected_type;- if(!field::get_type(obj, detected_type)) {- if(obj.is_null() && the_field.optional) {- // null values are allowed only if field is optional- return Option<bool>(true);+ bool found_dynamic_field = false;++ for(auto dyn_field_it = dyn_fields.begin(); dyn_field_it != dyn_fields.end(); dyn_field_it++) {+ auto& dynamic_field = dyn_field_it->second;++ if(dynamic_field.is_auto() || dynamic_field.is_string_star()) {+ continue;}- return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type.");+ if(std::regex_match(the_field.name, std::regex(dynamic_field.name))) {+ detected_type = dynamic_field.type;+ found_dynamic_field = true;+ break;+ }}- if(std::isalnum(detected_type.back()) && has_array) {- // convert singular type to multi valued type- detected_type += "[]";+ if(!found_dynamic_field) {+ if(!field::get_type(obj, detected_type)) {+ if(obj.is_null() && the_field.optional) {+ // null values are allowed only if field is optional+ return Option<bool>(true);+ }++ return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type.");+ }++ if(std::isalnum(detected_type.back()) && has_array) {+ // convert singular type to multi valued type+ detected_type += "[]";+ }}has_obj_array = has_obj_array || ((detected_type == field_types::OBJECT) && has_array);@@ -883,7 +924,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, consif(detected_type == the_field.type || is_numericaly_valid) {if(the_field.is_object()) {- flatten_obj(doc, obj, has_array, has_obj_array, the_field, the_field.name, flattened_fields);+ flatten_obj(doc, obj, has_array, has_obj_array, the_field, the_field.name, dyn_fields, flattened_fields);} else {if(doc.count(the_field.name) != 0 && flattened_fields.find(the_field.name) == flattened_fields.end()) {return Option<bool>(true);@@ -926,14 +967,15 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, consfor(auto& ele: it.value()) {has_obj_array = has_obj_array || ele.is_object();Option<bool> op = flatten_field(doc, ele, the_field, path_parts, path_index + 1, has_array,- has_obj_array, flattened_fields);+ has_obj_array, dyn_fields, flattened_fields);if(!op.ok()) {return op;}}return Option<bool>(true);} else {- return flatten_field(doc, it.value(), the_field, path_parts, path_index + 1, has_array, has_obj_array, flattened_fields);+ return flatten_field(doc, it.value(), the_field, path_parts, path_index + 1, has_array, has_obj_array,+ dyn_fields, flattened_fields);}} {return Option<bool>(404, "Field `" + the_field.name + "` not found.");@@ -942,6 +984,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, consOption<bool> field::flatten_doc(nlohmann::json& document,const tsl::htrie_map<char, field>& nested_fields,+ const std::unordered_map<std::string, field>& dyn_fields,bool missing_is_ok, std::vector<field>& flattened_fields) {std::unordered_map<std::string, field> flattened_fields_map;@@ -955,7 +998,8 @@ Option<bool> field::flatten_doc(nlohmann::json& document,continue;}- auto op = flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);+ auto op = flatten_field(document, document, nested_field, field_parts, 0, false, false,+ dyn_fields, flattened_fields_map);if(op.ok()) {continue;}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex bf7a5529..9693e08d 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -65,7 +65,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -108,7 +108,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());expected_json = R"({@@ -135,14 +135,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("locations.address", field_types::OBJECT, false)};- ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok()); // must be of type object_array+ ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok()); // must be of type object_arraynested_fields = {field("locations.address", field_types::OBJECT_ARRAY, false)};flattened_fields.clear();- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());expected_json = R"({@@ -174,7 +174,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company.name", field_types::STRING, false)};- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());expected_json = R"({@@ -225,7 +225,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -241,7 +241,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -261,7 +261,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees.detail.tags", field_types::STRING_ARRAY, false),};- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());ASSERT_EQ(3, flattened_fields.size());std::sort(flattened_fields.begin(), flattened_fields.end(), [](field& a, field& b) {@@ -290,7 +290,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {std::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `locations` not found.", flatten_op.error());@@ -299,7 +299,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {};flattened_fields.clear();- flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);+ flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());}@@ -317,7 +317,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));std::vector<field> flattened_fields;- field::flatten_doc(stored_doc, schema, true, flattened_fields);+ field::flatten_doc(stored_doc, schema, {}, true, flattened_fields);ASSERT_EQ(3, stored_doc[".flat"].size());ASSERT_EQ(7, stored_doc.size());@@ -366,7 +366,7 @@ TEST_F(CollectionNestedFieldsTest, CompactNestedFields) {ASSERT_EQ(1, schema.count("location_addresses"));std::vector<field> flattened_fields;- field::flatten_doc(stored_doc, schema, true, flattened_fields);+ field::flatten_doc(stored_doc, schema, {}, true, flattened_fields);ASSERT_EQ(2, stored_doc["location_addresses.city"].size());ASSERT_EQ(2, stored_doc["location_addresses.street"].size());@@ -2005,6 +2005,50 @@ TEST_F(CollectionNestedFieldsTest, WildcardWithExplicitSchema) {ASSERT_EQ(1, results["found"].get<size_t>());}+TEST_F(CollectionNestedFieldsTest, DynamicFieldWithExplicitSchema) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "spec", "type": "object"},+ {"name": "spec\\..*\\.value", "type": "float"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "spec": {"number": {"value": 100}}+ })"_json;++ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());++ auto field_vec = coll1->get_fields();+ ASSERT_EQ(3, field_vec.size());+ ASSERT_EQ(field_types::FLOAT, field_vec[2].type);++ // with only explicit nested dynamic type+ schema = R"({+ "name": "coll2",+ "enable_nested_fields": true,+ "fields": [+ {"name": ".*", "type": "auto"},+ {"name": "spec\\..*\\.value", "type": "float"}+ ]+ })"_json;++ op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll2 = op.get();+ ASSERT_TRUE(coll2->add(doc1.dump(), CREATE).ok());++ field_vec = coll2->get_fields();+ ASSERT_EQ(4, field_vec.size());+ ASSERT_EQ(field_types::FLOAT, field_vec[3].type);+}+TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {nlohmann::json schema = R"({"name": "coll1",