Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Basic alter support for nested fields.
Kishore Nalland2daad2c3d3d
diff --git a/include/collection.h b/include/collection.hindex f0d471c1..1c1b0aae 100644--- a/include/collection.h+++ b/include/collection.h@@ -165,6 +165,7 @@ private:const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,std::vector<field>& new_fields,+ std::vector<field>& nested_fields_found,bool enable_nested_fields);static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,@@ -199,11 +200,12 @@ private:Option<bool> batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,- const bool do_validation);+ const tsl::htrie_map<char, field>& alter_nested_fields);Option<bool> validate_alter_payload(nlohmann::json& schema_changes,std::vector<field>& addition_fields,std::vector<field>& reindex_fields,+ tsl::htrie_map<char, field>& new_nested_fields,std::vector<field>& del_fields,std::string& fallback_field_type);@@ -216,8 +218,6 @@ private:void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;- static void remove_flat_field_values(nlohmann::json& document);-static void remove_flat_fields(nlohmann::json& document);bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,@@ -325,6 +325,8 @@ public:tsl::htrie_map<char, field> get_schema();+ tsl::htrie_map<char, field> get_nested_fields();+std::string get_default_sorting_field();Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,@@ -351,7 +353,8 @@ public:static void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs);- void batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out, size_t &num_indexed, const bool& write_docs, const bool& write_id);+ void batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out, size_t &num_indexed,+ const bool& return_doc, const bool& return_id);bool is_exceeding_memory_threshold() const;@@ -373,7 +376,7 @@ public:nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,const index_operation_t& operation=CREATE, const std::string& id="",const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT,- const bool& write_docs=false, const bool& write_id=false);+ const bool& return_doc=false, const bool& return_id=false);Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,diff --git a/include/field.h b/include/field.hindex 2f3532b6..4b948ae7 100644--- a/include/field.h+++ b/include/field.h@@ -405,7 +405,7 @@ struct field {static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,std::vector<field>& flattened_fields);- static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema);+ static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);};struct filter {diff --git a/src/collection.cpp b/src/collection.cppindex f5eef56b..f08a4d69 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -202,7 +202,7 @@ Option<nlohmann::json> Collection::add(const std::string & json_str,nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmann::json& document,const index_operation_t& operation, const std::string& id,- const DIRTY_VALUES& dirty_values, const bool& write_docs, const bool& write_id) {+ const DIRTY_VALUES& dirty_values, const bool& return_doc, const bool& return_id) {//LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();std::vector<index_record> index_records;@@ -248,13 +248,15 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexingif(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty()) {std::vector<field> new_fields;+ std::vector<field> nested_fields_found;std::unique_lock lock(mutex);Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,search_schema, dynamic_fields,nested_fields,fallback_field_type,- new_fields, enable_nested_fields);+ new_fields, nested_fields_found,+ enable_nested_fields);if(!new_fields_op.ok()) {record.index_failure(new_fields_op.code(), new_fields_op.error());}@@ -265,6 +267,10 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmafields.emplace_back(new_field);}+ for(auto& nested_field: nested_fields_found) {+ nested_fields.emplace(nested_field.name, nested_field);+ }+auto persist_op = persist_collection_meta();if(!persist_op.ok()) {record.index_failure(persist_op.code(), persist_op.error());@@ -280,7 +286,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmado_batched_index:if((i+1) % index_batch_size == 0 || i == json_lines.size()-1 || repeated_doc) {- batch_index(index_records, json_lines, num_indexed, write_docs, write_id);+ batch_index(index_records, json_lines, num_indexed, return_doc, return_id);// to return the document for the single doc add casesif(index_records.size() == 1) {@@ -305,7 +311,7 @@ bool Collection::is_exceeding_memory_threshold() const {}void Collection::batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out,- size_t &num_indexed, const bool& write_docs, const bool& write_id) {+ size_t &num_indexed, const bool& return_doc, const bool& return_id) {batch_index_in_memory(index_records);@@ -331,7 +337,7 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect} else {// remove flattened field values before storing on disk (.flat meta will be kept)- remove_flat_field_values(index_record.doc);+ remove_flat_fields(index_record.doc);const std::string& seq_id_str = std::to_string(index_record.seq_id);const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,@@ -354,10 +360,10 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect}res["success"] = index_record.indexed.ok();- if (write_docs & index_record.indexed.ok()) {+ if (return_doc & index_record.indexed.ok()) {res["document"] = index_record.is_update ? index_record.new_doc : index_record.doc;}- if (write_id & index_record.indexed.ok()) {+ if (return_id & index_record.indexed.ok()) {res["id"] = index_record.is_update ? index_record.new_doc["id"] : index_record.doc["id"];}if(!index_record.indexed.ok()) {@@ -3040,6 +3046,11 @@ tsl::htrie_map<char, field> Collection::get_schema() {return search_schema;};+tsl::htrie_map<char, field> Collection::get_nested_fields() {+ std::shared_lock lock(mutex);+ return nested_fields;+};+std::string Collection::get_meta_key(const std::string & collection_name) {return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;}@@ -3078,8 +3089,8 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,return Option<bool>(500, "Error while parsing stored document with sequence ID: " + seq_id_key);}- if(!raw_doc) {- field::flatten_stored_doc(document, search_schema);+ if(!raw_doc && enable_nested_fields) {+ field::flatten_stored_doc(document, nested_fields);}return Option<bool>(true);}@@ -3199,12 +3210,13 @@ Option<bool> Collection::persist_collection_meta() {Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,- const bool do_validation) {+ const tsl::htrie_map<char, field>& alter_nested_fields) {// Update schema with additions (deletions can only be made later)std::vector<field> new_fields;-tsl::htrie_map<char, field> schema_additions;+ std::vector<std::string> nested_field_names;+for(auto& f: alter_fields) {if(f.name == ".*") {fields.push_back(f);@@ -3219,6 +3231,15 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsnew_fields.push_back(f);}+ if(f.nested) {+ // We should add only root nested fields into `nested_fields` data structure+ if(alter_nested_fields.count(f.name) != 0) {+ nested_fields.emplace(f.name, f);+ }++ nested_field_names.push_back(f.name);+ }+fields.push_back(f);}@@ -3248,6 +3269,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsnlohmann::detail::error_handler_t::ignore));}+ if(enable_nested_fields) {+ field::flatten_stored_doc(document, nested_fields);+ }+index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);iter_batch.emplace_back(std::move(record));@@ -3265,7 +3290,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions,- fallback_field_type, token_separators, symbols_to_index, do_validation);+ fallback_field_type, token_separators, symbols_to_index, true);iter_batch.clear();}@@ -3322,11 +3347,12 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {std::vector<field> del_fields;std::vector<field> addition_fields;std::vector<field> reindex_fields;+ tsl::htrie_map<char, field> new_nested_fields;std::string this_fallback_field_type;auto validate_op = validate_alter_payload(alter_payload, addition_fields, reindex_fields,- del_fields, this_fallback_field_type);+ new_nested_fields, del_fields, this_fallback_field_type);if(!validate_op.ok()) {return validate_op;}@@ -3344,14 +3370,14 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {LOG(INFO) << "Processing field additions and deletions first...";}- auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, true);+ auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, new_nested_fields);if(!batch_alter_op.ok()) {return batch_alter_op;}if(!reindex_fields.empty()) {LOG(INFO) << "Processing field modifications now...";- batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, true);+ batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, new_nested_fields);if(!batch_alter_op.ok()) {return batch_alter_op;}@@ -3360,17 +3386,11 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {return Option<bool>(true);}-void Collection::remove_flat_field_values(nlohmann::json& document) {+void Collection::remove_flat_fields(nlohmann::json& document) {if(document.contains(".flat")) {for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {document.erase(flat_key);}- }-}--void Collection::remove_flat_fields(nlohmann::json& document) {- remove_flat_field_values(document);- if(document.contains(".flat")) {document.erase(".flat");}}@@ -3459,6 +3479,7 @@ void Collection::prune_doc(nlohmann::json& doc,Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,std::vector<field>& addition_fields,std::vector<field>& reindex_fields,+ tsl::htrie_map<char, field>& new_nested_fields,std::vector<field>& del_fields,std::string& fallback_field_type) {if(!schema_changes.is_object()) {@@ -3538,6 +3559,23 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,if(found_field) {del_fields.push_back(field_it.value());updated_search_schema.erase(field_it.key());++ // handle nested fields+ if(field_it.value().nested && enable_nested_fields) {+ std::vector<std::string> sub_fields;+ auto ex_op = extract_field_name(field_name, search_schema, sub_fields, false, enable_nested_fields);+ if(!ex_op.ok()) {+ return ex_op;+ }++ for(auto& sub_field: sub_fields) {+ auto sub_field_it = search_schema.find(sub_field);+ if(sub_field_it != search_schema.end()) {+ del_fields.push_back(sub_field_it.value());+ updated_search_schema.erase(sub_field);+ }+ }+ }}// NOTE: fields with type "auto" or "string*" will exist in both `search_schema` and `dynamic_fields`@@ -3584,6 +3622,11 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,} else {addition_fields.push_back(f);}++ if(f.nested) {+ new_nested_fields.emplace(f.name, f);+ }+} else {// partial update is not supported for nowreturn Option<bool>(400, "Field `" + field_name + "` is already part of the schema: To "@@ -3617,19 +3660,27 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,nlohmann::detail::error_handler_t::ignore));}- if(!fallback_field_type.empty() || !new_dynamic_fields.empty()) {+ if(enable_nested_fields) {+ field::flatten_stored_doc(document, nested_fields);+ }++ if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !new_nested_fields.empty()) {std::vector<field> new_fields;+ std::vector<field> nested_fields_found;+Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,updated_search_schema, new_dynamic_fields,- nested_fields,+ new_nested_fields,fallback_field_type,- new_fields, enable_nested_fields);+ new_fields, nested_fields_found,+ enable_nested_fields);if(!new_fields_op.ok()) {return new_fields_op;}for(auto& new_field: new_fields) {updated_search_schema[new_field.name] = new_field;+ // NOTE: cannot be pushed to `addition_fields` to support drop+re-addition use casereindex_fields.push_back(new_field);}}@@ -3783,10 +3834,9 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,std::vector<field>& new_fields,+ std::vector<field>& nested_fields_found,const bool enable_nested_fields) {- std::vector<field> nested_fields_found;-if(enable_nested_fields) {for(auto& nested_field: nested_fields) {nested_fields_found.push_back(nested_field);diff --git a/src/field.cpp b/src/field.cppindex 6d41c877..3c57fda9 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -730,19 +730,9 @@ Option<bool> field::flatten_doc(nlohmann::json& document,return Option<bool>(true);}-Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema) {- if(document.count(".flat") == 0) {- return Option<bool>(true);- }-+Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields) {std::unordered_map<std::string, field> flattened_fields_map;- for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {- auto field_it = schema.find(flat_key);- if(field_it == schema.end()) {- continue;- }-- const field& nested_field = field_it.value();+ for(const auto& nested_field: nested_fields) {std::vector<std::string> field_parts;StringUtils::split(nested_field.name, field_parts, ".");@@ -754,6 +744,14 @@ Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htriflatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);}+ if(document.count(".flat") == 0) {+ document[".flat"] = nlohmann::json::array();+ }++ for(auto& kv: flattened_fields_map) {+ document[".flat"].push_back(kv.second.name);+ }+return Option<bool>(true);}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 09caa17d..e09f57e9 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -293,7 +293,6 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {auto stored_doc = R"({- ".flat": ["employees.num", "details.name", "details.year"],"employees": { "num": 1200 },"foo": "bar","details": [{"name": "foo", "year": 2000}]@@ -312,16 +311,6 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {ASSERT_EQ(1, stored_doc.count("employees.num"));ASSERT_EQ(1, stored_doc.count("details.name"));ASSERT_EQ(1, stored_doc.count("details.year"));-- // stored doc without any .flat- stored_doc = R"({- "employees": { "num": 1200 },- "foo": "bar",- "details": [{"name": "foo", "year": 2000}]- })"_json;-- field::flatten_stored_doc(stored_doc, schema);- ASSERT_EQ(3, stored_doc.size());}TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {@@ -384,12 +373,12 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {ASSERT_EQ(0, results["hits"][0].count(".flat"));ASSERT_EQ(0, results["hits"][0].count("employees.tags"));- // raw document in the store will have only the .flat meta key but not actual flat fields+ // raw document in the store will not have the .flat meta key or actual flat fieldsnlohmann::json raw_doc;coll1->get_document_from_store(0, raw_doc, true);- ASSERT_EQ(1, raw_doc.count(".flat"));+ ASSERT_EQ(0, raw_doc.count(".flat"));ASSERT_EQ(0, raw_doc.count("employees.tags"));- ASSERT_EQ(5, raw_doc.size());+ ASSERT_EQ(4, raw_doc.size());ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex eb102cf1..91b65b68 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1197,3 +1197,143 @@ TEST_F(CollectionSchemaChangeTest, DropIntegerFieldAndAddStringValues) {ASSERT_TRUE(res_op.ok());ASSERT_EQ(1, res_op.get()["found"].get<size_t>());}++TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {+ // Plain object field+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "person", "type": "object"},+ {"name": "school.city", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "Test";+ doc["person"] = nlohmann::json::object();+ doc["person"]["name"] = "Jack";+ doc["school"] = nlohmann::json::object();+ doc["school"]["city"] = "NYC";++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ auto fields = coll1->get_fields();+ auto schema_map = coll1->get_schema();++ ASSERT_EQ(4, fields.size());+ ASSERT_EQ(4, schema_map.size());++ // drop object field++ auto schema_changes = R"({+ "fields": [+ {"name": "person", "drop": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ fields = coll1->get_fields();+ schema_map = coll1->get_schema();++ ASSERT_EQ(2, fields.size());+ ASSERT_EQ(2, schema_map.size());++ // drop primitive nested field++ schema_changes = R"({+ "fields": [+ {"name": "school.city", "drop": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ fields = coll1->get_fields();+ schema_map = coll1->get_schema();++ ASSERT_EQ(1, fields.size());+ ASSERT_EQ(1, schema_map.size());+}++TEST_F(CollectionSchemaChangeTest, NestedFieldSchemaAdditions) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "Test";+ doc["person"] = nlohmann::json::object();+ doc["person"]["name"] = "Jack";+ doc["school"] = nlohmann::json::object();+ doc["school"]["city"] = "NYC";+ doc["school"]["state"] = "NY";++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ auto fields = coll1->get_fields();+ auto schema_map = coll1->get_schema();++ ASSERT_EQ(1, fields.size());+ ASSERT_EQ(1, schema_map.size());+ ASSERT_EQ(0, coll1->get_nested_fields().size());++ // add plain object field++ auto schema_changes = R"({+ "fields": [+ {"name": "person", "type": "object"}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ fields = coll1->get_fields();+ schema_map = coll1->get_schema();++ ASSERT_EQ(3, fields.size());+ ASSERT_EQ(3, schema_map.size());+ ASSERT_EQ(1, coll1->get_nested_fields().size());++ // nested primitive field++ schema_changes = R"({+ "fields": [+ {"name": "school.city", "type": "string"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ fields = coll1->get_fields();+ schema_map = coll1->get_schema();++ ASSERT_EQ(4, fields.size());+ ASSERT_EQ(4, schema_map.size());+ ASSERT_EQ(2, coll1->get_nested_fields().size());++ // try searching on new fields+ auto res_op = coll1->search("jack", {"person.name"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);+ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());++ res_op = coll1->search("nyc", {"school.city"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);+ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());+}