Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Generify removal of field with different type due to auto schema detection.
Kishore Nallan0edee3ae1212
diff --git a/src/index.cpp b/src/index.cppindex 62b7c570..fb0ff7cf 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4858,12 +4858,6 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, constreturn;}- if(document[field_name].is_object() && !search_field.is_object()) {- // Handle an edge case with auto schema detection:- // Value as object field (ignored), but later as string field will cause documents with incorrect on-disk type- return ;- }-// Go through all the field names and find the keys+values so that they can be removed from in-memory indexif(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {std::vector<std::string> tokens;@@ -4987,18 +4981,34 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docconst std::vector<field>& del_fields, const bool is_update) {std::unique_lock lock(mutex);+ // The exception during removal is mostly because of an edge case with auto schema detection:+ // Value indexed as Type T but later if field is dropped and reindexed in another type X,+ // the on-disk data will differ from the newly detected type on schema. We've to log the error,+ // but have to ignore the field and proceed because there's no leak caused here.+if(!del_fields.empty()) {for(auto& the_field: del_fields) {if(!document.contains(the_field.name)) {// could be an optional fieldcontinue;}- remove_field(seq_id, document, the_field.name);++ try {+ remove_field(seq_id, document, the_field.name);+ } catch(const std::exception& e) {+ LOG(WARNING) << "Error while removing field `" << the_field.name << "` from document, message: "+ << e.what();+ }}} else {for(auto it = document.begin(); it != document.end(); ++it) {const std::string& field_name = it.key();- remove_field(seq_id, document, field_name);+ try {+ remove_field(seq_id, document, field_name);+ } catch(const std::exception& e) {+ LOG(WARNING) << "Error while removing field `" << field_name << "` from document, message: "+ << e.what();+ }}}diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 867e101f..b0a9390f 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -801,6 +801,33 @@ TEST_F(CollectionSchemaChangeTest, DropFieldNotExistingInDocuments) {ASSERT_TRUE(alter_op.ok());}+TEST_F(CollectionSchemaChangeTest, ChangeFieldToCoercableTypeIsNotAllowed) {+ // optional title field+ std::vector<field> fields = {field("title", field_types::STRING, false, true, true, "", 1, 1),+ field("points", field_types::INT32, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "").get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["points"] = 100;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // coerce field from int to string+ auto schema_changes = R"({+ "fields": [+ {"name": "points", "drop": true},+ {"name": "points", "type": "string"}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Schema change is incompatible with the type of documents already stored in this collection. "+ "Existing data for field `points` cannot be coerced into a string.", alter_op.error());+}+TEST_F(CollectionSchemaChangeTest, ChangeFromPrimitiveToDynamicField) {nlohmann::json req_json = R"({"name": "coll1",@@ -1061,4 +1088,76 @@ TEST_F(CollectionSchemaChangeTest, IndexFalseToTrue) {ASSERT_TRUE(res_op.ok());ASSERT_EQ(1, res_op.get()["found"].get<size_t>());ASSERT_EQ(1, res_op.get()["facet_counts"].size());-}\ No newline at end of file+}++TEST_F(CollectionSchemaChangeTest, DropIntegerFieldAndAddStringValues) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": ".*", "type": "auto"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ // index a label field as integer++ nlohmann::json doc;+ doc["id"] = "0";+ doc["label"] = 1000;+ doc["title"] = "Foo";+ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // drop this field from schema+ auto schema_changes = R"({+ "fields": [+ {"name": "label", "drop": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ // add new document with a string label+ doc["id"] = "1";+ doc["label"] = "abcdef";+ doc["title"] = "Bar";+ add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // now we have documents which have both string and integer for the same field :BOOM:+ // schema change operation should not be allowed at this point+ schema_changes = R"({+ "fields": [+ {"name": "year", "type": "int32", "optional": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Schema change is incompatible with the type of documents already stored in this collection. "+ "Existing data for field `label` cannot be coerced into a string.", alter_op.error());++ // but should allow the problematic field to be dropped+ schema_changes = R"({+ "fields": [+ {"name": "label", "drop": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ // add document with another field+ doc["id"] = "2";+ doc["label"] = "xyz";+ doc["year"] = 1947;+ add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // try searching for string label+ auto res_op = coll1->search("xyz", {"label"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);+ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());+}