Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix alter on dynamic fields.
Kishore Nallan284c3d0dc7f7
diff --git a/include/collection.h b/include/collection.hindex 457c3495..bb084c2f 100644--- a/include/collection.h+++ b/include/collection.h@@ -165,6 +165,7 @@ private:Option<bool> persist_collection_meta();Option<bool> batch_alter_data(const std::unordered_map<std::string, field>& schema_additions,+ const std::unordered_map<std::string, field>& new_dynamic_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,const bool do_validation);@@ -172,6 +173,8 @@ private:Option<bool> validate_alter_payload(nlohmann::json& schema_changes,std::unordered_map<std::string, field>& schema_additions,std::unordered_map<std::string, field>& schema_reindex,+ std::unordered_map<std::string, field>& addition_dynamic_fields,+ std::unordered_map<std::string, field>& reindex_dynamic_fields,std::vector<field>& del_fields,std::string& fallback_field_type);diff --git a/src/collection.cpp b/src/collection.cppindex e6b3ead1..e5ee2c4b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2535,6 +2535,7 @@ Option<bool> Collection::persist_collection_meta() {}Option<bool> Collection::batch_alter_data(const std::unordered_map<std::string, field>& schema_additions,+ const std::unordered_map<std::string, field>& new_dynamic_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,const bool do_validation) {@@ -2549,14 +2550,15 @@ Option<bool> Collection::batch_alter_data(const std::unordered_map<std::string,continue;}- if(f.is_dynamic()) {- // regexp fields and fields with auto type are treated as dynamic fields- dynamic_fields.emplace(f.name, f);- } else {- search_schema.emplace(kv.first, f);- new_fields.push_back(f);- }+ search_schema.emplace(kv.first, f);+ new_fields.push_back(f);+ fields.push_back(f);+ }+ for(auto& kv: new_dynamic_fields) {+ // regexp fields and fields with auto type are treated as dynamic fields+ const auto& f = kv.second;+ dynamic_fields.emplace(f.name, f);fields.push_back(f);}@@ -2659,10 +2661,13 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {// Validate that all stored documents are compatible with the proposed schema changes.std::unordered_map<std::string, field> schema_additions;std::unordered_map<std::string, field> schema_reindex;+ std::unordered_map<std::string, field> addition_dynamic_fields;+ std::unordered_map<std::string, field> reindex_dynamic_fields;std::vector<field> del_fields;std::string this_fallback_field_type;auto validate_op = validate_alter_payload(alter_payload, schema_additions, schema_reindex,+ addition_dynamic_fields, reindex_dynamic_fields,del_fields, this_fallback_field_type);if(!validate_op.ok()) {return validate_op;@@ -2681,7 +2686,7 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {LOG(INFO) << "Processing field additions and deletions first...";}- auto batch_alter_op = batch_alter_data(schema_additions, del_fields, fallback_field_type, false);+ auto batch_alter_op = batch_alter_data(schema_additions, addition_dynamic_fields, del_fields, fallback_field_type, false);if(!batch_alter_op.ok()) {return batch_alter_op;}@@ -2691,7 +2696,7 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {// we've to run revaliation because during schema change, some coercion might be needed// e.g. "123" -> 123 (string to integer)bool do_validation = true;- batch_alter_op = batch_alter_data(schema_reindex, {}, fallback_field_type, do_validation);+ batch_alter_op = batch_alter_data(schema_reindex, reindex_dynamic_fields, {}, fallback_field_type, do_validation);if(!batch_alter_op.ok()) {return batch_alter_op;}@@ -2703,6 +2708,8 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,std::unordered_map<std::string, field>& schema_additions,std::unordered_map<std::string, field>& schema_reindex,+ std::unordered_map<std::string, field>& addition_dynamic_fields,+ std::unordered_map<std::string, field>& reindex_dynamic_fields,std::vector<field>& del_fields,std::string& fallback_field_type) {if(!schema_changes.is_object()) {@@ -2753,7 +2760,8 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,auto found_field = (field_it != search_schema.end());auto dyn_field_it = dynamic_fields.find(field_name);- auto found_dyn_field = (dyn_field_it != dynamic_fields.end());+ auto found_dyn_field = (dyn_field_it != dynamic_fields.end()) ||+ (found_field && field_types::is_string_or_array(field_it->second.type));if(kv.value().contains("drop")) {if(!kv.value()["drop"].is_boolean() || !kv.value()["drop"].get<bool>()) {@@ -2771,7 +2779,17 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,if(found_field) {del_fields.push_back(field_it->second);- } else if(found_dyn_field) {+ updated_search_schema.erase(field_it->first);++ // we will also have to resolve the dynamic field names which match the static field name+ for(auto& field_kv: dynamic_fields) {+ if(std::regex_match(field_kv.first, std::regex(field_it->first))) {+ del_fields.push_back(field_kv.second);+ }+ }+ }++ else if(found_dyn_field) {del_fields.push_back(dyn_field_it->second);// we will also have to resolve the actual field names which match the dynamic field patternfor(auto& field_kv: search_schema) {@@ -2805,18 +2823,20 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,if(f.is_dynamic()) {new_dynamic_fields.emplace(f.name, f);++ if(is_reindex) {+ reindex_dynamic_fields.emplace(f.name, f);+ } else {+ addition_dynamic_fields.emplace(f.name, f);+ }} else {updated_search_schema[f.name] = f;- }-- if(is_reindex) {- // delete + reindex is fine, we will handle these fields separately- //if(!f.is_dynamic()) {- // expanded versions of dynamic fields will be discovered during data iteration below+ if(is_reindex) {+ // delete + reindex: we will handle these fields separatelyschema_reindex.emplace(f.name, f);- //}- } else {- schema_additions.emplace(f.name, f);+ } else {+ schema_additions.emplace(f.name, f);+ }}} else {// partial update is not supported for now@@ -2851,7 +2871,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,nlohmann::detail::error_handler_t::ignore));}- if(!fallback_field_type.empty() || !new_dynamic_fields.empty()) {+ if(!fallback_field_type.empty() || !addition_dynamic_fields.empty() || !reindex_dynamic_fields.empty()) {std::vector<field> new_fields;Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,updated_search_schema, new_dynamic_fields,@@ -2862,8 +2882,8 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}for(auto& new_field: new_fields) {- updated_search_schema.emplace(new_field.name, new_field);- schema_reindex.emplace(new_field.name, new_field);+ updated_search_schema[new_field.name] = new_field;+ schema_reindex[new_field.name] = new_field;}}diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex aa584a68..df858c5c 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -452,6 +452,19 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {ASSERT_EQ("The `fields` value should be an array of objects containing the field `name` ""and other properties.", alter_op.error());+ // 8. sending full collection schema, like creation body+ schema_changes = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Only `fields` can be updated at the moment.",alter_op.error());+collectionManager.drop_collection("coll1");}@@ -741,4 +754,187 @@ TEST_F(CollectionSchemaChangeTest, DropFieldNotExistingInDocuments) {auto alter_op = coll1->alter(schema_changes);ASSERT_TRUE(alter_op.ok());-}\ No newline at end of file+}++TEST_F(CollectionSchemaChangeTest, ChangeFromPrimitiveToDynamicField) {+ nlohmann::json req_json = R"({+ "name": "coll1",+ "fields": [+ {"name": "tags", "type": "string"}+ ]+ })"_json;++ auto coll1_op = collectionManager.create_collection(req_json);+ ASSERT_TRUE(coll1_op.ok());++ auto coll1 = coll1_op.get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["tags"] = "123";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(1, coll1->get_fields().size());+ ASSERT_EQ(0, coll1->get_dynamic_fields().size());++ // try to alter to string* type++ auto schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "string*", "facet": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ auto results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_dynamic_fields().size());++ // go back to plain string type+ schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "string", "facet": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(1, coll1->get_fields().size());+ ASSERT_EQ(0, coll1->get_dynamic_fields().size());+}++TEST_F(CollectionSchemaChangeTest, ChangeFromPrimitiveToAutoField) {+ nlohmann::json req_json = R"({+ "name": "coll1",+ "fields": [+ {"name": "tags", "type": "string"}+ ]+ })"_json;++ auto coll1_op = collectionManager.create_collection(req_json);+ ASSERT_TRUE(coll1_op.ok());++ auto coll1 = coll1_op.get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["tags"] = "123";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(1, coll1->get_fields().size());+ ASSERT_EQ(0, coll1->get_dynamic_fields().size());++ // try to alter to auto type++ auto schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "auto", "facet": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ auto results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_dynamic_fields().size());++ // go back to plain string type+ schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "string", "facet": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(1, coll1->get_fields().size());+ ASSERT_EQ(0, coll1->get_dynamic_fields().size());+}++TEST_F(CollectionSchemaChangeTest, ChangeFromStringStarToAutoField) {+ nlohmann::json req_json = R"({+ "name": "coll1",+ "fields": [+ {"name": "tags", "type": "string*"}+ ]+ })"_json;++ auto coll1_op = collectionManager.create_collection(req_json);+ ASSERT_TRUE(coll1_op.ok());++ auto coll1 = coll1_op.get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["tags"] = "123";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_dynamic_fields().size());++ // try to alter to auto type++ auto schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "auto", "facet": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ auto results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_dynamic_fields().size());++ // go back to string* type+ schema_changes = R"({+ "fields": [+ {"name": "tags", "drop": true},+ {"name": "tags", "type": "string*", "facet": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("123", {"tags"}, "", {"tags"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ ASSERT_EQ(1, coll1->get_schema().size());+ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_dynamic_fields().size());+}