Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support nested dynamic fields.

Kishore Nallan806008f42143
diff --git a/include/field.h b/include/field.h
index d4361db9..4fa6cc6e 100644
--- a/include/field.h
+++ b/include/field.h
@@ -489,13 +489,17 @@ struct field {
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
const field& the_field, const std::string& flat_name,
+ const std::unordered_map<std::string, field>& dyn_fields,
std::unordered_map<std::string, field>& flattened_fields);
static Option<bool> flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,
std::vector<std::string>& path_parts, size_t path_index, bool has_array,
- bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);
+ bool has_obj_array,
+ const std::unordered_map<std::string, field>& dyn_fields,
+ std::unordered_map<std::string, field>& flattened_fields);
static Option<bool> flatten_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields,
+ const std::unordered_map<std::string, field>& dyn_fields,
bool missing_is_ok, std::vector<field>& flattened_fields);
static void compact_nested_fields(tsl::htrie_map<char, field>& nested_fields);
diff --git a/src/collection.cpp b/src/collection.cpp
index e4737326..90ca8b1a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3511,7 +3511,7 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
if(!raw_doc && enable_nested_fields) {
std::vector<field> flattened_fields;
- field::flatten_doc(document, nested_fields, true, flattened_fields);
+ field::flatten_doc(document, nested_fields, {}, true, flattened_fields);
}
return Option<bool>(true);
@@ -3700,7 +3700,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
if(enable_nested_fields) {
std::vector<field> flattened_fields;
- field::flatten_doc(document, nested_fields, true, flattened_fields);
+ field::flatten_doc(document, nested_fields, {}, true, flattened_fields);
}
index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);
@@ -4383,13 +4383,6 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
auto& dynamic_field = dyn_field_it->second;
if(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {
- // unless the field is auto or string*, ignore field name matching regexp pattern
- if(kv.key() == dynamic_field.name && !dynamic_field.is_auto() &&
- !dynamic_field.is_string_star()) {
- skip_field = true;
- break;
- }
-
// to prevent confusion we also disallow dynamic field names that contain ".*"
if((kv.key() != ".*" && kv.key().find(".*") != std::string::npos)) {
skip_field = true;
@@ -4437,7 +4430,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
}
std::vector<field> flattened_fields;
- auto flatten_op = field::flatten_doc(document, nested_fields, is_update, flattened_fields);
+ auto flatten_op = field::flatten_doc(document, nested_fields, dyn_fields, is_update, flattened_fields);
if(!flatten_op.ok()) {
return flatten_op;
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 28bd0f5a..52b6ce15 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1322,7 +1322,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
if(collection->get_enable_nested_fields()) {
std::vector<field> flattened_fields;
- field::flatten_doc(document, collection->get_nested_fields(), true, flattened_fields);
+ field::flatten_doc(document, collection->get_nested_fields(), {}, true, flattened_fields);
}
auto dirty_values = DIRTY_VALUES::DROP;
diff --git a/src/field.cpp b/src/field.cpp
index 6c572b99..bcc8470a 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -4,6 +4,7 @@
#include "text_embedder_manager.h"
#include <stack>
#include <collection_manager.h>
+#include <regex>
Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,
const std::string& format_err_msg,
@@ -802,15 +803,17 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
const field& the_field, const std::string& flat_name,
+ const std::unordered_map<std::string, field>& dyn_fields,
std::unordered_map<std::string, field>& flattened_fields) {
if(value.is_object()) {
has_obj_array = has_array;
for(const auto& kv: value.items()) {
- flatten_obj(doc, kv.value(), has_array, has_obj_array, the_field, flat_name + "." + kv.key(), flattened_fields);
+ flatten_obj(doc, kv.value(), has_array, has_obj_array, the_field, flat_name + "." + kv.key(),
+ dyn_fields, flattened_fields);
}
} else if(value.is_array()) {
for(const auto& kv: value.items()) {
- flatten_obj(doc, kv.value(), true, has_obj_array, the_field, flat_name, flattened_fields);
+ flatten_obj(doc, kv.value(), true, has_obj_array, the_field, flat_name, dyn_fields, flattened_fields);
}
} else { // must be a primitive
if(doc.count(flat_name) != 0 && flattened_fields.find(flat_name) == flattened_fields.end()) {
@@ -818,8 +821,31 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arr
}
std::string detected_type;
- if(!field::get_type(value, detected_type)) {
- return false;
+ bool found_dynamic_field = false;
+
+ for(auto dyn_field_it = dyn_fields.begin(); dyn_field_it != dyn_fields.end(); dyn_field_it++) {
+ auto& dynamic_field = dyn_field_it->second;
+
+ if(dynamic_field.is_auto() || dynamic_field.is_string_star()) {
+ continue;
+ }
+
+ if(std::regex_match(flat_name, std::regex(flat_name))) {
+ detected_type = dynamic_field.type;
+ found_dynamic_field = true;
+ break;
+ }
+ }
+
+ if(!found_dynamic_field) {
+ if(!field::get_type(value, detected_type)) {
+ return false;
+ }
+
+ if(std::isalnum(detected_type.back()) && has_array) {
+ // convert singular type to multi valued type
+ detected_type += "[]";
+ }
}
if(has_array) {
@@ -828,11 +854,6 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arr
doc[flat_name] = value;
}
- if(std::isalnum(detected_type.back()) && has_array) {
- // convert singular type to multi valued type
- detected_type += "[]";
- }
-
field flattened_field = the_field;
flattened_field.name = flat_name;
flattened_field.type = detected_type;
@@ -848,22 +869,42 @@ bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_arr
Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,
std::vector<std::string>& path_parts, size_t path_index,
- bool has_array, bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields) {
+ bool has_array, bool has_obj_array,
+ const std::unordered_map<std::string, field>& dyn_fields,
+ std::unordered_map<std::string, field>& flattened_fields) {
if(path_index == path_parts.size()) {
// end of path: check if obj matches expected type
std::string detected_type;
- if(!field::get_type(obj, detected_type)) {
- if(obj.is_null() && the_field.optional) {
- // null values are allowed only if field is optional
- return Option<bool>(true);
+ bool found_dynamic_field = false;
+
+ for(auto dyn_field_it = dyn_fields.begin(); dyn_field_it != dyn_fields.end(); dyn_field_it++) {
+ auto& dynamic_field = dyn_field_it->second;
+
+ if(dynamic_field.is_auto() || dynamic_field.is_string_star()) {
+ continue;
}
- return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type.");
+ if(std::regex_match(the_field.name, std::regex(dynamic_field.name))) {
+ detected_type = dynamic_field.type;
+ found_dynamic_field = true;
+ break;
+ }
}
- if(std::isalnum(detected_type.back()) && has_array) {
- // convert singular type to multi valued type
- detected_type += "[]";
+ if(!found_dynamic_field) {
+ if(!field::get_type(obj, detected_type)) {
+ if(obj.is_null() && the_field.optional) {
+ // null values are allowed only if field is optional
+ return Option<bool>(true);
+ }
+
+ return Option<bool>(400, "Field `" + the_field.name + "` has an incorrect type.");
+ }
+
+ if(std::isalnum(detected_type.back()) && has_array) {
+ // convert singular type to multi valued type
+ detected_type += "[]";
+ }
}
has_obj_array = has_obj_array || ((detected_type == field_types::OBJECT) && has_array);
@@ -883,7 +924,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
if(detected_type == the_field.type || is_numericaly_valid) {
if(the_field.is_object()) {
- flatten_obj(doc, obj, has_array, has_obj_array, the_field, the_field.name, flattened_fields);
+ flatten_obj(doc, obj, has_array, has_obj_array, the_field, the_field.name, dyn_fields, flattened_fields);
} else {
if(doc.count(the_field.name) != 0 && flattened_fields.find(the_field.name) == flattened_fields.end()) {
return Option<bool>(true);
@@ -926,14 +967,15 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
for(auto& ele: it.value()) {
has_obj_array = has_obj_array || ele.is_object();
Option<bool> op = flatten_field(doc, ele, the_field, path_parts, path_index + 1, has_array,
- has_obj_array, flattened_fields);
+ has_obj_array, dyn_fields, flattened_fields);
if(!op.ok()) {
return op;
}
}
return Option<bool>(true);
} else {
- return flatten_field(doc, it.value(), the_field, path_parts, path_index + 1, has_array, has_obj_array, flattened_fields);
+ return flatten_field(doc, it.value(), the_field, path_parts, path_index + 1, has_array, has_obj_array,
+ dyn_fields, flattened_fields);
}
} {
return Option<bool>(404, "Field `" + the_field.name + "` not found.");
@@ -942,6 +984,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
Option<bool> field::flatten_doc(nlohmann::json& document,
const tsl::htrie_map<char, field>& nested_fields,
+ const std::unordered_map<std::string, field>& dyn_fields,
bool missing_is_ok, std::vector<field>& flattened_fields) {
std::unordered_map<std::string, field> flattened_fields_map;
@@ -955,7 +998,8 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
continue;
}
- auto op = flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);
+ auto op = flatten_field(document, document, nested_field, field_parts, 0, false, false,
+ dyn_fields, flattened_fields_map);
if(op.ok()) {
continue;
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index bf7a5529..9693e08d 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -65,7 +65,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -108,7 +108,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
expected_json = R"(
{
@@ -135,14 +135,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("locations.address", field_types::OBJECT, false)
};
- ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok()); // must be of type object_array
+ ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok()); // must be of type object_array
nested_fields = {
field("locations.address", field_types::OBJECT_ARRAY, false)
};
flattened_fields.clear();
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
expected_json = R"(
{
@@ -174,7 +174,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company.name", field_types::STRING, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
expected_json = R"(
{
@@ -225,7 +225,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -241,7 +241,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -261,7 +261,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees.detail.tags", field_types::STRING_ARRAY, false),
};
- ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields).ok());
ASSERT_EQ(3, flattened_fields.size());
std::sort(flattened_fields.begin(), flattened_fields.end(), [](field& a, field& b) {
@@ -290,7 +290,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);
+ auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `locations` not found.", flatten_op.error());
@@ -299,7 +299,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
};
flattened_fields.clear();
- flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);
+ flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), {}, false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());
}
@@ -317,7 +317,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));
std::vector<field> flattened_fields;
- field::flatten_doc(stored_doc, schema, true, flattened_fields);
+ field::flatten_doc(stored_doc, schema, {}, true, flattened_fields);
ASSERT_EQ(3, stored_doc[".flat"].size());
ASSERT_EQ(7, stored_doc.size());
@@ -366,7 +366,7 @@ TEST_F(CollectionNestedFieldsTest, CompactNestedFields) {
ASSERT_EQ(1, schema.count("location_addresses"));
std::vector<field> flattened_fields;
- field::flatten_doc(stored_doc, schema, true, flattened_fields);
+ field::flatten_doc(stored_doc, schema, {}, true, flattened_fields);
ASSERT_EQ(2, stored_doc["location_addresses.city"].size());
ASSERT_EQ(2, stored_doc["location_addresses.street"].size());
@@ -2005,6 +2005,50 @@ TEST_F(CollectionNestedFieldsTest, WildcardWithExplicitSchema) {
ASSERT_EQ(1, results["found"].get<size_t>());
}
+TEST_F(CollectionNestedFieldsTest, DynamicFieldWithExplicitSchema) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "spec", "type": "object"},
+ {"name": "spec\\..*\\.value", "type": "float"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "spec": {"number": {"value": 100}}
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
+
+ auto field_vec = coll1->get_fields();
+ ASSERT_EQ(3, field_vec.size());
+ ASSERT_EQ(field_types::FLOAT, field_vec[2].type);
+
+ // with only explicit nested dynamic type
+ schema = R"({
+ "name": "coll2",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": ".*", "type": "auto"},
+ {"name": "spec\\..*\\.value", "type": "float"}
+ ]
+ })"_json;
+
+ op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll2 = op.get();
+ ASSERT_TRUE(coll2->add(doc1.dump(), CREATE).ok());
+
+ field_vec = coll2->get_fields();
+ ASSERT_EQ(4, field_vec.size());
+ ASSERT_EQ(field_types::FLOAT, field_vec[3].type);
+}
+
TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
nlohmann::json schema = R"({
"name": "coll1",