Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support nested fields with explicit schema.
Kishore Nallanf86f8b54872a
diff --git a/include/collection.h b/include/collection.hindex 9b42ece0..0036cb28 100644--- a/include/collection.h+++ b/include/collection.h@@ -229,6 +229,17 @@ private:const tsl::htrie_map<char, field>& search_schema,std::vector<std::string>& processed_search_fields);+ static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,+ const tsl::htrie_map<char, field>& schema,+ std::vector<field>& new_fields);++ bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;++ template<class T>+ static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,+ const nlohmann::json& fdoc, nlohmann::json& fobj,+ std::vector<std::string>& path_parts, size_t path_index, T func);+public:enum {MAX_ARRAY_MATCHES = 5};@@ -457,7 +468,7 @@ public:};template<class T>-bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,+bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,const nlohmann::json& fdoc, nlohmann::json& fobj,std::vector<std::string>& path_parts, size_t path_index, T func) {if(path_index == path_parts.size()) {diff --git a/include/field.h b/include/field.hindex 7a0a94fa..c32d3b32 100644--- a/include/field.h+++ b/include/field.h@@ -43,6 +43,8 @@ namespace fields {static const std::string sort = "sort";static const std::string infix = "infix";static const std::string locale = "locale";+ static const std::string nested = "nested";+ static const std::string nested_array = "nested_array";}struct field {@@ -56,14 +58,21 @@ struct field {bool infix;bool nested; // field inside an object- bool nested_array; // field inside an array of objects that is forced to be an array++ // field inside an array of objects that is forced to be an array+ // integer to handle tri-state: true (1), false (0), not known yet (2)+ // third state is used to diff between array of object and array within object during write+ int nested_array;++ static constexpr int VAL_UNKNOWN = 2;field() {}field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,- bool index = true, std::string locale = "", int sort = -1, int infix = -1) :+ bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,+ int nested_array = 0) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),- nested(false), nested_array(false) {+ nested(nested), nested_array(nested_array) {if(sort != -1) {this->sort = bool(sort);@@ -155,11 +164,17 @@ struct field {}bool is_dynamic() const {- return is_dynamic(name, type);+ return is_dynamic_type() || nested;+ }++ static bool is_dynamic_type(const std::string& name, const std::string& type) {+ return type == "string*" || (name != ".*" && type == field_types::AUTO) ||+ type == field_types::OBJECT || type == field_types::OBJECT_ARRAY ||+ (name != ".*" && name.find(".*") != std::string::npos);}- static bool is_dynamic(const std::string& name, const std::string& type) {- return type == "string*" || (name != ".*" && type == field_types::AUTO) || (name != ".*" && name.find(".*") != std::string::npos);+ bool is_dynamic_type() const {+ return is_dynamic_type(name, type);}bool has_numerical_index() const {@@ -295,7 +310,7 @@ struct field {found_default_sorting_field = true;}- if(field.is_dynamic() && !field.optional) {+ if(field.is_dynamic() && !field.nested && !field.optional) {return Option<bool>(400, "Field `" + field.name + "` must be an optional field.");}diff --git a/src/collection.cpp b/src/collection.cppindex 04e2c85c..142041e2 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2093,6 +2093,23 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_treturn true;}+bool Collection::is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const {+ auto child_it = obj.find(path_parts[part_i]);+ if(child_it == obj.end()) {+ return false;+ }++ if(child_it.value().is_array() && !child_it.value().empty() && child_it.value().at(0).is_object()) {+ return true;+ }++ if(part_i+1 == path_parts.size()) {+ return false;+ }++ return is_nested_array(child_it.value(), path_parts, part_i+1);+}+void Collection::highlight_result(const std::string& raw_query, const field &search_field,const size_t search_field_index,const tsl::htrie_map<char, token_leaf>& qtoken_leaves,@@ -2176,9 +2193,18 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaindexed offsets.*/- if(search_field.nested_array) {- std::vector<std::string> path_parts;+ int nested_array = search_field.nested_array;+ std::vector<std::string> path_parts;++ if(nested_array == field::VAL_UNKNOWN) {StringUtils::split(search_field.name, path_parts, ".");+ nested_array = Collection::is_nested_array(document, path_parts, 0);+ }++ if(nested_array) {+ if(path_parts.empty()) {+ StringUtils::split(search_field.name, path_parts, ".");+ }highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,path_parts, 0, [&](nlohmann::json& h_obj, nlohmann::json& f_obj) {@@ -3537,7 +3563,16 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,// check against dynamic field definitionsfor(const auto& dynamic_field: dyn_fields) {- if(std::regex_match (kv.key(), std::regex(dynamic_field.first))) {+ if (dynamic_field.second.nested && (kv.key() == dynamic_field.first ||+ (StringUtils::begins_with(dynamic_field.first, kv.key()) &&+ dynamic_field.first[kv.key().size()] == '.'))) {+ new_field = dynamic_field.second;+ new_field.name = dynamic_field.first;+ found_dynamic_field = true;+ break;+ }++ else if(std::regex_match (kv.key(), std::regex(dynamic_field.first))) {// unless the field is auto or string*, ignore field name matching regexp patternif(kv.key() == dynamic_field.first && !dynamic_field.second.is_auto() &&!dynamic_field.second.is_string_star()) {@@ -3620,6 +3655,9 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,if(test_field_type == field_types::AUTO) {new_field.type = field_type;+ if(new_field.is_object()) {+ new_field.nested = true;+ }} else {if (kv.value().is_array()) {new_field.type = field_types::STRING_ARRAY;@@ -3639,7 +3677,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,new_field.sort = true;}- if(new_field.is_object()) {+ if(new_field.nested) {nested_fields.push_back(new_field);} else {new_fields.emplace_back(new_field);@@ -3649,8 +3687,18 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,kv++;}+ return flatten_and_identify_new_fields(document, nested_fields, schema, new_fields);+}++Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields,+ const tsl::htrie_map<char, field>& schema,+ std::vector<field>& new_fields) {+ if(nested_fields.empty()) {+ return Option<bool>(true);+ }+std::vector<field> flattened_fields;- auto flatten_op = field::flatten_doc(document, nested_fields, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);if(!flatten_op.ok()) {return flatten_op;}@@ -3669,6 +3717,9 @@ Index* Collection::init_index() {if(field.is_dynamic()) {// regexp fields and fields with auto type are treated as dynamic fieldsdynamic_fields.emplace(field.name, field);+ if(!field.is_dynamic_type()) {+ search_schema.emplace(field.name, field);+ }continue;}diff --git a/src/field.cpp b/src/field.cppindex f5929894..a59c5b2c 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -509,15 +509,27 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<}if(field_json.count(fields::optional) == 0) {- // dynamic fields are always optional- bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);+ // dynamic type fields are always optional+ bool is_dynamic = field::is_dynamic_type(field_json[fields::name], field_json[fields::type]);field_json[fields::optional] = is_dynamic;}+ bool is_obj = field_json[fields::type] == field_types::OBJECT || field_json[fields::type] == field_types::OBJECT_ARRAY;+ bool is_regexp_name = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;++ if(is_obj || (!is_regexp_name && field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {+ field_json[fields::nested] = true;+ field_json[fields::nested_array] = field::VAL_UNKNOWN; // unknown, will be resolved during read+ } else {+ field_json[fields::nested] = false;+ field_json[fields::nested_array] = 0;+ }+the_fields.emplace_back(field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],field_json[fields::optional], field_json[fields::index], field_json[fields::locale],- field_json[fields::sort], field_json[fields::infix])+ field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],+ field_json[fields::nested_array]));return Option<bool>(true);diff --git a/src/index.cpp b/src/index.cppindex c17702ca..3182ca22 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4849,7 +4849,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecstd::unique_lock lock(mutex);for(const auto & new_field: new_fields) {- if(new_field.is_dynamic() || !new_field.index) {+ if(!new_field.index || new_field.is_dynamic_type()) {continue;}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 3d63b1bc..867468f0 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -717,15 +717,187 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"].size());ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());- ASSERT_EQ("brown", results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"][0]);- ASSERT_EQ("fox", results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"][1]);+ std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();+ std::sort(matched_tokens.begin(), matched_tokens.end());+ ASSERT_EQ("brown", matched_tokens[0]);+ ASSERT_EQ("fox", matched_tokens[1]);ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].size());- ASSERT_EQ("brown", results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"][0]);- ASSERT_EQ("fox", results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"][1]);+ matched_tokens = results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].get<std::vector<std::string>>();+ std::sort(matched_tokens.begin(), matched_tokens.end());+ ASSERT_EQ("brown", matched_tokens[0]);+ ASSERT_EQ("fox", matched_tokens[1]);ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].size());- ASSERT_EQ("Brown", results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"][0]);+ matched_tokens = results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].get<std::vector<std::string>>();+ std::sort(matched_tokens.begin(), matched_tokens.end());+ ASSERT_EQ("Brown", matched_tokens[0]);+}++TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "details", "type": "object", "optional": false },+ {"name": "company.name", "type": "string", "optional": false },+ {"name": "locations", "type": "object[]", "optional": false }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc = R"({+ "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],+ "details": {+ "description": "Quick set, go.",+ "names": ["Quick brown fox jumped.", "The red fox was not fast."]+ },+ "company": {"name": "Quick and easy fix."},+ "locations": [+ {+ "address": { "street": "Brown Shade Avenue" }+ },+ {+ "address": { "street": "Graywolf Lane" }+ }+ ]+ })"_json;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto sop = coll1->search("brown fox", {"details", "locations"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4);++ // search both simply nested and deeply nested array-of-objects+ auto results = coll1->search("brown fox", {"details", "locations"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();++ auto snippet_doc = R"({+ "details":{+ "names":[+ "Quick <mark>brown</mark> <mark>fox</mark> jumped.",+ "The red <mark>fox</mark> was not fast."+ ]+ },+ "locations":[+ {+ "address":{+ "street":"<mark>Brown</mark> Shade Avenue"+ }+ },+ {+ "address":{+ "street":"Graywolf Lane"+ }+ }+ ]+ })"_json;++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());++ sop = coll1->search("fix", {"company.name"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4);+++ results = coll1->search("fix", {"company.name"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();++ ASSERT_EQ(1, results["hits"].size());++ // explicit nested array field (locations.address.street)+ schema = R"({+ "name": "coll2",+ "fields": [+ {"name": "details", "type": "object", "optional": false },+ {"name": "company.name", "type": "string", "optional": false },+ {"name": "locations.address.street", "type": "string[]", "optional": false }+ ]+ })"_json;++ op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll2 = op.get();++ add_op = coll2->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ results = coll2->search("brown", {"locations.address.street"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();++ ASSERT_EQ(1, results["hits"].size());++ snippet_doc = R"({+ "locations":[+ {+ "address":{+ "street":"<mark>Brown</mark> Shade Avenue"+ }+ },+ {+ "address":{+ "street":"Graywolf Lane"+ }+ }+ ]+ })"_json;++ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());++ // explicit partial array object field in the schema+ schema = R"({+ "name": "coll3",+ "fields": [+ {"name": "details", "type": "object", "optional": false },+ {"name": "company.name", "type": "string", "optional": false },+ {"name": "locations.address", "type": "object[]", "optional": false }+ ]+ })"_json;++ op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll3 = op.get();++ add_op = coll3->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ results = coll3->search("brown", {"locations.address"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();++ ASSERT_EQ(1, results["hits"].size());++ snippet_doc = R"({+ "locations":[+ {+ "address":{+ "street":"<mark>Brown</mark> Shade Avenue"+ }+ },+ {+ "address":{+ "street":"Graywolf Lane"+ }+ }+ ]+ })"_json;++ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+}TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {