Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix update of nested fields.

Kishore Nallanf17c7d3d5f28
diff --git a/src/index.cpp b/src/index.cpp
index c7a33928..6426b4ee 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -6132,18 +6132,36 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
void Index::get_doc_changes(const index_operation_t op, nlohmann::json& update_doc,
const nlohmann::json& old_doc, nlohmann::json& new_doc, nlohmann::json& del_doc) {
- // construct new doc with old doc values first
+ if(op == UPSERT) {
+ new_doc = update_doc;
+ } else {
+ new_doc = old_doc;
+ new_doc.merge_patch(update_doc);
+ if(old_doc.contains(".flat")) {
+ new_doc[".flat"] = old_doc[".flat"];
+ for(auto& fl: update_doc[".flat"]) {
+ new_doc[".flat"].push_back(fl);
+ }
+ }
+ }
+
for(auto it = old_doc.begin(); it != old_doc.end(); ++it) {
+ if(it.value().is_object() || (it.value().is_array() && (it.value().empty() || it.value()[0].is_object()))) {
+ continue;
+ }
+
if(op == UPSERT && !update_doc.contains(it.key())) {
del_doc[it.key()] = it.value();
- } else {
- new_doc[it.key()] = it.value();
}
}
- // now override new doc with updated doc values and also create del doc
auto it = update_doc.begin();
for(; it != update_doc.end(); ) {
+ if(it.value().is_object() || (it.value().is_array() && (it.value().empty() || it.value()[0].is_object()))) {
+ ++it;
+ continue;
+ }
+
// if the update doc contains a field that exists in old, we record that (for delete + reindex)
bool field_exists_in_old_doc = (old_doc.count(it.key()) != 0);
if(field_exists_in_old_doc) {
@@ -6158,7 +6176,6 @@ void Index::get_doc_changes(const index_operation_t op, nlohmann::json& update_d
new_doc.erase(it.key());
it = update_doc.erase(it);
} else {
- new_doc[it.key()] = it.value();
++it;
}
}
@@ -6186,6 +6203,10 @@ void Index::scrub_reindex_doc(const tsl::htrie_map<char, field>& search_schema,
continue;
}
+ if(it.value().is_object() || (it.value().is_array() && (it.value().empty() || it.value()[0].is_object()))) {
+ continue;
+ }
+
const auto search_field = search_field_it.value(); // copy, don't use reference!
// compare values between old and update docs:
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 7aa3d931..80fc61db 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -2050,6 +2050,119 @@ TEST_F(CollectionNestedFieldsTest, DynamicFieldWithExplicitSchema) {
}
TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name":"name", "type": "string", "index": false, "optional": true},
+ {"name":"brand","type":"object","optional":true},
+ {"name":"brand.id","type":"int32","sort":false},
+ {"name":"brand.name","type":"string","index":false,"sort":false,"optional":true}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "product_id": 63992305,
+ "name": "Chips",
+ "link": "http://wicked-uncle.biz",
+ "meta": {
+ "valid": true
+ },
+ "brand": {
+ "id": 34002,
+ "name": "Hodkiewicz - Rempel"
+ }
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
+
+ // action=update - `name` changed, `id` not sent
+ // `id` field should not be deleted
+
+ auto doc_update = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "brand": {
+ "name": "Rempel"
+ }
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc_update.dump(), UPDATE).ok());
+
+ auto results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"][0]["document"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["brand"].size());
+ ASSERT_EQ("Rempel", results["hits"][0]["document"]["brand"]["name"].get<std::string>());
+
+ // action=emplace
+ doc_update = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "brand": {
+ "name": "The Rempel"
+ }
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc_update.dump(), EMPLACE).ok());
+
+ results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"][0]["document"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["brand"].size());
+ ASSERT_EQ("The Rempel", results["hits"][0]["document"]["brand"]["name"].get<std::string>());
+
+ // action=upsert requires the full document
+ doc_update = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "brand": {
+ "name": "Xomel"
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc_update.dump(), UPSERT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `brand.id` has been declared in the schema, but is not found in the document.", add_op.error());
+
+ doc_update = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "name": "Chips",
+ "brand": {
+ "id": 34002,
+ "name": "Xomel"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc_update.dump(), UPSERT);
+ ASSERT_TRUE(add_op.ok());
+ results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["brand"].size());
+ ASSERT_EQ("Xomel", results["hits"][0]["document"]["brand"]["name"].get<std::string>());
+
+ // upsert with brand.name missing is allowed because it's optional
+ doc_update = R"({
+ "id": "b4db5f0456a93320428365f92c2a54ce15df2d0a",
+ "name": "Potato Chips",
+ "brand": {
+ "id": 34002
+ }
+ })"_json;
+
+ add_op = coll1->add(doc_update.dump(), UPSERT);
+ ASSERT_TRUE(add_op.ok());
+ results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+ ASSERT_EQ(1, results["hits"][0]["document"]["brand"].size());
+ ASSERT_EQ(34002, results["hits"][0]["document"]["brand"]["id"].get<size_t>());
+}
+
+TEST_F(CollectionNestedFieldsTest, UpdateOfNestFieldsWithWildcardSchema) {
nlohmann::json schema = R"({
"name": "coll1",
"enable_nested_fields": true,
@@ -2092,10 +2205,10 @@ TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
results = coll1->search("beta", {"studies.name"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
- // try removing fields via upsert
+ // try removing fields via upsert, dropping "company.year"
doc_update = R"({
"id": "0",
- "company": {"num_employees": 2000, "founded": 1976},
+ "company": {"num_employees": 4000, "founded": 1976},
"studies": [{"name": "College Alpha"}]
})"_json;
ASSERT_TRUE(coll1->add(doc_update.dump(), UPSERT).ok());
@@ -2110,10 +2223,11 @@ TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
ASSERT_EQ(0, results["found"].get<size_t>());
results = coll1->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(3, results["hits"][0]["document"].size());
ASSERT_EQ(2, results["hits"][0]["document"]["company"].size());
- ASSERT_EQ(2000, results["hits"][0]["document"]["company"]["num_employees"].get<size_t>());
+ ASSERT_EQ(4000, results["hits"][0]["document"]["company"]["num_employees"].get<size_t>());
ASSERT_EQ(1976, results["hits"][0]["document"]["company"]["founded"].get<size_t>());
ASSERT_EQ(1, results["hits"][0]["document"]["studies"].size());
ASSERT_EQ(1, results["hits"][0]["document"]["studies"][0].size());
@@ -2129,6 +2243,13 @@ TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
results = coll1->search("*", {}, "company.founded: 1976", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["company"].size());
+ ASSERT_EQ(2000, results["hits"][0]["document"]["company"]["num_employees"].get<size_t>());
+ ASSERT_EQ(1976, results["hits"][0]["document"]["company"]["founded"].get<size_t>());
+ ASSERT_EQ(1, results["hits"][0]["document"]["studies"].size());
+ ASSERT_EQ(1, results["hits"][0]["document"]["studies"][0].size());
+ ASSERT_EQ("College Alpha", results["hits"][0]["document"]["studies"][0]["name"].get<std::string>());
// via emplace (should not remove, since document can be partial)
doc_update = R"({
@@ -2140,6 +2261,14 @@ TEST_F(CollectionNestedFieldsTest, UpdateOfNestFields) {
results = coll1->search("*", {}, "company.num_employees: 2000", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["company"].size());
+ ASSERT_EQ(2000, results["hits"][0]["document"]["company"]["num_employees"].get<size_t>());
+ ASSERT_EQ(1976, results["hits"][0]["document"]["company"]["founded"].get<size_t>());
+ ASSERT_EQ(1, results["hits"][0]["document"]["studies"].size());
+ ASSERT_EQ(2, results["hits"][0]["document"]["studies"][0].size());
+ ASSERT_EQ("College Alpha", results["hits"][0]["document"]["studies"][0]["name"].get<std::string>());
+ ASSERT_EQ(1977, results["hits"][0]["document"]["studies"][0]["year"].get<size_t>());
}
TEST_F(CollectionNestedFieldsTest, NestedSchemaWithSingularType) {