Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support nested fields with explicit schema.

Kishore Nallanf86f8b54872a
diff --git a/include/collection.h b/include/collection.h
index 9b42ece0..0036cb28 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -229,6 +229,17 @@ private:
const tsl::htrie_map<char, field>& search_schema,
std::vector<std::string>& processed_search_fields);
+ static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,
+ const tsl::htrie_map<char, field>& schema,
+ std::vector<field>& new_fields);
+
+ bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;
+
+ template<class T>
+ static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
+ const nlohmann::json& fdoc, nlohmann::json& fobj,
+ std::vector<std::string>& path_parts, size_t path_index, T func);
+
public:
enum {MAX_ARRAY_MATCHES = 5};
@@ -457,7 +468,7 @@ public:
};
template<class T>
-bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
+bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
const nlohmann::json& fdoc, nlohmann::json& fobj,
std::vector<std::string>& path_parts, size_t path_index, T func) {
if(path_index == path_parts.size()) {
diff --git a/include/field.h b/include/field.h
index 7a0a94fa..c32d3b32 100644
--- a/include/field.h
+++ b/include/field.h
@@ -43,6 +43,8 @@ namespace fields {
static const std::string sort = "sort";
static const std::string infix = "infix";
static const std::string locale = "locale";
+ static const std::string nested = "nested";
+ static const std::string nested_array = "nested_array";
}
struct field {
@@ -56,14 +58,21 @@ struct field {
bool infix;
bool nested; // field inside an object
- bool nested_array; // field inside an array of objects that is forced to be an array
+
+ // field inside an array of objects that is forced to be an array
+ // integer to handle tri-state: true (1), false (0), not known yet (2)
+ // third state is used to diff between array of object and array within object during write
+ int nested_array;
+
+ static constexpr int VAL_UNKNOWN = 2;
field() {}
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
- bool index = true, std::string locale = "", int sort = -1, int infix = -1) :
+ bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,
+ int nested_array = 0) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),
- nested(false), nested_array(false) {
+ nested(nested), nested_array(nested_array) {
if(sort != -1) {
this->sort = bool(sort);
@@ -155,11 +164,17 @@ struct field {
}
bool is_dynamic() const {
- return is_dynamic(name, type);
+ return is_dynamic_type() || nested;
+ }
+
+ static bool is_dynamic_type(const std::string& name, const std::string& type) {
+ return type == "string*" || (name != ".*" && type == field_types::AUTO) ||
+ type == field_types::OBJECT || type == field_types::OBJECT_ARRAY ||
+ (name != ".*" && name.find(".*") != std::string::npos);
}
- static bool is_dynamic(const std::string& name, const std::string& type) {
- return type == "string*" || (name != ".*" && type == field_types::AUTO) || (name != ".*" && name.find(".*") != std::string::npos);
+ bool is_dynamic_type() const {
+ return is_dynamic_type(name, type);
}
bool has_numerical_index() const {
@@ -295,7 +310,7 @@ struct field {
found_default_sorting_field = true;
}
- if(field.is_dynamic() && !field.optional) {
+ if(field.is_dynamic() && !field.nested && !field.optional) {
return Option<bool>(400, "Field `" + field.name + "` must be an optional field.");
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 04e2c85c..142041e2 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2093,6 +2093,23 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
return true;
}
+bool Collection::is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const {
+ auto child_it = obj.find(path_parts[part_i]);
+ if(child_it == obj.end()) {
+ return false;
+ }
+
+ if(child_it.value().is_array() && !child_it.value().empty() && child_it.value().at(0).is_object()) {
+ return true;
+ }
+
+ if(part_i+1 == path_parts.size()) {
+ return false;
+ }
+
+ return is_nested_array(child_it.value(), path_parts, part_i+1);
+}
+
void Collection::highlight_result(const std::string& raw_query, const field &search_field,
const size_t search_field_index,
const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
@@ -2176,9 +2193,18 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
indexed offsets.
*/
- if(search_field.nested_array) {
- std::vector<std::string> path_parts;
+ int nested_array = search_field.nested_array;
+ std::vector<std::string> path_parts;
+
+ if(nested_array == field::VAL_UNKNOWN) {
StringUtils::split(search_field.name, path_parts, ".");
+ nested_array = Collection::is_nested_array(document, path_parts, 0);
+ }
+
+ if(nested_array) {
+ if(path_parts.empty()) {
+ StringUtils::split(search_field.name, path_parts, ".");
+ }
highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,
path_parts, 0, [&](nlohmann::json& h_obj, nlohmann::json& f_obj) {
@@ -3537,7 +3563,16 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
// check against dynamic field definitions
for(const auto& dynamic_field: dyn_fields) {
- if(std::regex_match (kv.key(), std::regex(dynamic_field.first))) {
+ if (dynamic_field.second.nested && (kv.key() == dynamic_field.first ||
+ (StringUtils::begins_with(dynamic_field.first, kv.key()) &&
+ dynamic_field.first[kv.key().size()] == '.'))) {
+ new_field = dynamic_field.second;
+ new_field.name = dynamic_field.first;
+ found_dynamic_field = true;
+ break;
+ }
+
+ else if(std::regex_match (kv.key(), std::regex(dynamic_field.first))) {
// unless the field is auto or string*, ignore field name matching regexp pattern
if(kv.key() == dynamic_field.first && !dynamic_field.second.is_auto() &&
!dynamic_field.second.is_string_star()) {
@@ -3620,6 +3655,9 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
if(test_field_type == field_types::AUTO) {
new_field.type = field_type;
+ if(new_field.is_object()) {
+ new_field.nested = true;
+ }
} else {
if (kv.value().is_array()) {
new_field.type = field_types::STRING_ARRAY;
@@ -3639,7 +3677,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
new_field.sort = true;
}
- if(new_field.is_object()) {
+ if(new_field.nested) {
nested_fields.push_back(new_field);
} else {
new_fields.emplace_back(new_field);
@@ -3649,8 +3687,18 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
kv++;
}
+ return flatten_and_identify_new_fields(document, nested_fields, schema, new_fields);
+}
+
+Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,
+ const tsl::htrie_map<char, field>& schema,
+ std::vector<field>& new_fields) {
+ if(nested_fields.empty()) {
+ return Option<bool>(true);
+ }
+
std::vector<field> flattened_fields;
- auto flatten_op = field::flatten_doc(document, nested_fields, flattened_fields);
+ auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);
if(!flatten_op.ok()) {
return flatten_op;
}
@@ -3669,6 +3717,9 @@ Index* Collection::init_index() {
if(field.is_dynamic()) {
// regexp fields and fields with auto type are treated as dynamic fields
dynamic_fields.emplace(field.name, field);
+ if(!field.is_dynamic_type()) {
+ search_schema.emplace(field.name, field);
+ }
continue;
}
diff --git a/src/field.cpp b/src/field.cpp
index f5929894..a59c5b2c 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -509,15 +509,27 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<
}
if(field_json.count(fields::optional) == 0) {
- // dynamic fields are always optional
- bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
+ // dynamic type fields are always optional
+ bool is_dynamic = field::is_dynamic_type(field_json[fields::name], field_json[fields::type]);
field_json[fields::optional] = is_dynamic;
}
+ bool is_obj = field_json[fields::type] == field_types::OBJECT || field_json[fields::type] == field_types::OBJECT_ARRAY;
+ bool is_regexp_name = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;
+
+ if(is_obj || (!is_regexp_name && field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {
+ field_json[fields::nested] = true;
+ field_json[fields::nested_array] = field::VAL_UNKNOWN; // unknown, will be resolved during read
+ } else {
+ field_json[fields::nested] = false;
+ field_json[fields::nested_array] = 0;
+ }
+
the_fields.emplace_back(
field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
- field_json[fields::sort], field_json[fields::infix])
+ field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
+ field_json[fields::nested_array])
);
return Option<bool>(true);
diff --git a/src/index.cpp b/src/index.cpp
index c17702ca..3182ca22 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -4849,7 +4849,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
std::unique_lock lock(mutex);
for(const auto & new_field: new_fields) {
- if(new_field.is_dynamic() || !new_field.index) {
+ if(!new_field.index || new_field.is_dynamic_type()) {
continue;
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 3d63b1bc..867468f0 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -717,15 +717,187 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"].size());
ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());
- ASSERT_EQ("brown", results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"][0]);
- ASSERT_EQ("fox", results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"][1]);
+ std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();
+ std::sort(matched_tokens.begin(), matched_tokens.end());
+ ASSERT_EQ("brown", matched_tokens[0]);
+ ASSERT_EQ("fox", matched_tokens[1]);
ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].size());
- ASSERT_EQ("brown", results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"][0]);
- ASSERT_EQ("fox", results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"][1]);
+ matched_tokens = results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].get<std::vector<std::string>>();
+ std::sort(matched_tokens.begin(), matched_tokens.end());
+ ASSERT_EQ("brown", matched_tokens[0]);
+ ASSERT_EQ("fox", matched_tokens[1]);
ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].size());
- ASSERT_EQ("Brown", results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"][0]);
+ matched_tokens = results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].get<std::vector<std::string>>();
+ std::sort(matched_tokens.begin(), matched_tokens.end());
+ ASSERT_EQ("Brown", matched_tokens[0]);
+}
+
+TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "details", "type": "object", "optional": false },
+ {"name": "company.name", "type": "string", "optional": false },
+ {"name": "locations", "type": "object[]", "optional": false }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc = R"({
+ "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],
+ "details": {
+ "description": "Quick set, go.",
+ "names": ["Quick brown fox jumped.", "The red fox was not fast."]
+ },
+ "company": {"name": "Quick and easy fix."},
+ "locations": [
+ {
+ "address": { "street": "Brown Shade Avenue" }
+ },
+ {
+ "address": { "street": "Graywolf Lane" }
+ }
+ ]
+ })"_json;
+
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto sop = coll1->search("brown fox", {"details", "locations"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4);
+
+ // search both simply nested and deeply nested array-of-objects
+ auto results = coll1->search("brown fox", {"details", "locations"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
+
+ auto snippet_doc = R"({
+ "details":{
+ "names":[
+ "Quick <mark>brown</mark> <mark>fox</mark> jumped.",
+ "The red <mark>fox</mark> was not fast."
+ ]
+ },
+ "locations":[
+ {
+ "address":{
+ "street":"<mark>Brown</mark> Shade Avenue"
+ }
+ },
+ {
+ "address":{
+ "street":"Graywolf Lane"
+ }
+ }
+ ]
+ })"_json;
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+
+ sop = coll1->search("fix", {"company.name"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4);
+
+
+ results = coll1->search("fix", {"company.name"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ // explicit nested array field (locations.address.street)
+ schema = R"({
+ "name": "coll2",
+ "fields": [
+ {"name": "details", "type": "object", "optional": false },
+ {"name": "company.name", "type": "string", "optional": false },
+ {"name": "locations.address.street", "type": "string[]", "optional": false }
+ ]
+ })"_json;
+
+ op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll2 = op.get();
+
+ add_op = coll2->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll2->search("brown", {"locations.address.street"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ snippet_doc = R"({
+ "locations":[
+ {
+ "address":{
+ "street":"<mark>Brown</mark> Shade Avenue"
+ }
+ },
+ {
+ "address":{
+ "street":"Graywolf Lane"
+ }
+ }
+ ]
+ })"_json;
+
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+
+ // explicit partial array object field in the schema
+ schema = R"({
+ "name": "coll3",
+ "fields": [
+ {"name": "details", "type": "object", "optional": false },
+ {"name": "company.name", "type": "string", "optional": false },
+ {"name": "locations.address", "type": "object[]", "optional": false }
+ ]
+ })"_json;
+
+ op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll3 = op.get();
+
+ add_op = coll3->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll3->search("brown", {"locations.address"},
+ "", {}, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ snippet_doc = R"({
+ "locations":[
+ {
+ "address":{
+ "street":"<mark>Brown</mark> Shade Avenue"
+ }
+ },
+ {
+ "address":{
+ "street":"Graywolf Lane"
+ }
+ }
+ ]
+ })"_json;
+
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+
}
TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {