Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Generify removal of field with different type due to auto schema detection.

Kishore Nallan0edee3ae1212
diff --git a/src/index.cpp b/src/index.cpp
index 62b7c570..fb0ff7cf 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -4858,12 +4858,6 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
return;
}
- if(document[field_name].is_object() && !search_field.is_object()) {
- // Handle an edge case with auto schema detection:
- // Value as object field (ignored), but later as string field will cause documents with incorrect on-disk type
- return ;
- }
-
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
std::vector<std::string> tokens;
@@ -4987,18 +4981,34 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
const std::vector<field>& del_fields, const bool is_update) {
std::unique_lock lock(mutex);
+ // The exception during removal is mostly because of an edge case with auto schema detection:
+ // Value indexed as Type T but later if field is dropped and reindexed in another type X,
+ // the on-disk data will differ from the newly detected type on schema. We've to log the error,
+ // but have to ignore the field and proceed because there's no leak caused here.
+
if(!del_fields.empty()) {
for(auto& the_field: del_fields) {
if(!document.contains(the_field.name)) {
// could be an optional field
continue;
}
- remove_field(seq_id, document, the_field.name);
+
+ try {
+ remove_field(seq_id, document, the_field.name);
+ } catch(const std::exception& e) {
+ LOG(WARNING) << "Error while removing field `" << the_field.name << "` from document, message: "
+ << e.what();
+ }
}
} else {
for(auto it = document.begin(); it != document.end(); ++it) {
const std::string& field_name = it.key();
- remove_field(seq_id, document, field_name);
+ try {
+ remove_field(seq_id, document, field_name);
+ } catch(const std::exception& e) {
+ LOG(WARNING) << "Error while removing field `" << field_name << "` from document, message: "
+ << e.what();
+ }
}
}
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 867e101f..b0a9390f 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -801,6 +801,33 @@ TEST_F(CollectionSchemaChangeTest, DropFieldNotExistingInDocuments) {
ASSERT_TRUE(alter_op.ok());
}
+TEST_F(CollectionSchemaChangeTest, ChangeFieldToCoercableTypeIsNotAllowed) {
+ // optional title field
+ std::vector<field> fields = {field("title", field_types::STRING, false, true, true, "", 1, 1),
+ field("points", field_types::INT32, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "").get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // coerce field from int to string
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "points", "drop": true},
+ {"name": "points", "type": "string"}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Schema change is incompatible with the type of documents already stored in this collection. "
+ "Existing data for field `points` cannot be coerced into a string.", alter_op.error());
+}
+
TEST_F(CollectionSchemaChangeTest, ChangeFromPrimitiveToDynamicField) {
nlohmann::json req_json = R"({
"name": "coll1",
@@ -1061,4 +1088,76 @@ TEST_F(CollectionSchemaChangeTest, IndexFalseToTrue) {
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
ASSERT_EQ(1, res_op.get()["facet_counts"].size());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionSchemaChangeTest, DropIntegerFieldAndAddStringValues) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": ".*", "type": "auto"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ // index a label field as integer
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["label"] = 1000;
+ doc["title"] = "Foo";
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // drop this field from schema
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "label", "drop": true}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ // add new document with a string label
+ doc["id"] = "1";
+ doc["label"] = "abcdef";
+ doc["title"] = "Bar";
+ add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // now we have documents which have both string and integer for the same field :BOOM:
+ // schema change operation should not be allowed at this point
+ schema_changes = R"({
+ "fields": [
+ {"name": "year", "type": "int32", "optional": true}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Schema change is incompatible with the type of documents already stored in this collection. "
+ "Existing data for field `label` cannot be coerced into a string.", alter_op.error());
+
+ // but should allow the problematic field to be dropped
+ schema_changes = R"({
+ "fields": [
+ {"name": "label", "drop": true}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ // add document with another field
+ doc["id"] = "2";
+ doc["label"] = "xyz";
+ doc["year"] = 1947;
+ add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // try searching for string label
+ auto res_op = coll1->search("xyz", {"label"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
+}