Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Basic alter support for nested fields.

Kishore Nalland2daad2c3d3d
diff --git a/include/collection.h b/include/collection.h
index f0d471c1..1c1b0aae 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -165,6 +165,7 @@ private:
const tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
std::vector<field>& new_fields,
+ std::vector<field>& nested_fields_found,
bool enable_nested_fields);
static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,
@@ -199,11 +200,12 @@ private:
Option<bool> batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type,
- const bool do_validation);
+ const tsl::htrie_map<char, field>& alter_nested_fields);
Option<bool> validate_alter_payload(nlohmann::json& schema_changes,
std::vector<field>& addition_fields,
std::vector<field>& reindex_fields,
+ tsl::htrie_map<char, field>& new_nested_fields,
std::vector<field>& del_fields,
std::string& fallback_field_type);
@@ -216,8 +218,6 @@ private:
void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;
- static void remove_flat_field_values(nlohmann::json& document);
-
static void remove_flat_fields(nlohmann::json& document);
bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,
@@ -325,6 +325,8 @@ public:
tsl::htrie_map<char, field> get_schema();
+ tsl::htrie_map<char, field> get_nested_fields();
+
std::string get_default_sorting_field();
Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,
@@ -351,7 +353,8 @@ public:
static void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs);
- void batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out, size_t &num_indexed, const bool& write_docs, const bool& write_id);
+ void batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out, size_t &num_indexed,
+ const bool& return_doc, const bool& return_id);
bool is_exceeding_memory_threshold() const;
@@ -373,7 +376,7 @@ public:
nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
const index_operation_t& operation=CREATE, const std::string& id="",
const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT,
- const bool& write_docs=false, const bool& write_id=false);
+ const bool& return_doc=false, const bool& return_id=false);
Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
diff --git a/include/field.h b/include/field.h
index 2f3532b6..4b948ae7 100644
--- a/include/field.h
+++ b/include/field.h
@@ -405,7 +405,7 @@ struct field {
static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,
std::vector<field>& flattened_fields);
- static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema);
+ static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);
};
struct filter {
diff --git a/src/collection.cpp b/src/collection.cpp
index f5eef56b..f08a4d69 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -202,7 +202,7 @@ Option<nlohmann::json> Collection::add(const std::string & json_str,
nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
const index_operation_t& operation, const std::string& id,
- const DIRTY_VALUES& dirty_values, const bool& write_docs, const bool& write_id) {
+ const DIRTY_VALUES& dirty_values, const bool& return_doc, const bool& return_id) {
//LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();
std::vector<index_record> index_records;
@@ -248,13 +248,15 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing
if(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty()) {
std::vector<field> new_fields;
+ std::vector<field> nested_fields_found;
std::unique_lock lock(mutex);
Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,
search_schema, dynamic_fields,
nested_fields,
fallback_field_type,
- new_fields, enable_nested_fields);
+ new_fields, nested_fields_found,
+ enable_nested_fields);
if(!new_fields_op.ok()) {
record.index_failure(new_fields_op.code(), new_fields_op.error());
}
@@ -265,6 +267,10 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
fields.emplace_back(new_field);
}
+ for(auto& nested_field: nested_fields_found) {
+ nested_fields.emplace(nested_field.name, nested_field);
+ }
+
auto persist_op = persist_collection_meta();
if(!persist_op.ok()) {
record.index_failure(persist_op.code(), persist_op.error());
@@ -280,7 +286,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
do_batched_index:
if((i+1) % index_batch_size == 0 || i == json_lines.size()-1 || repeated_doc) {
- batch_index(index_records, json_lines, num_indexed, write_docs, write_id);
+ batch_index(index_records, json_lines, num_indexed, return_doc, return_id);
// to return the document for the single doc add cases
if(index_records.size() == 1) {
@@ -305,7 +311,7 @@ bool Collection::is_exceeding_memory_threshold() const {
}
void Collection::batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out,
- size_t &num_indexed, const bool& write_docs, const bool& write_id) {
+ size_t &num_indexed, const bool& return_doc, const bool& return_id) {
batch_index_in_memory(index_records);
@@ -331,7 +337,7 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect
} else {
// remove flattened field values before storing on disk (.flat meta will be kept)
- remove_flat_field_values(index_record.doc);
+ remove_flat_fields(index_record.doc);
const std::string& seq_id_str = std::to_string(index_record.seq_id);
const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,
@@ -354,10 +360,10 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect
}
res["success"] = index_record.indexed.ok();
- if (write_docs & index_record.indexed.ok()) {
+ if (return_doc & index_record.indexed.ok()) {
res["document"] = index_record.is_update ? index_record.new_doc : index_record.doc;
}
- if (write_id & index_record.indexed.ok()) {
+ if (return_id & index_record.indexed.ok()) {
res["id"] = index_record.is_update ? index_record.new_doc["id"] : index_record.doc["id"];
}
if(!index_record.indexed.ok()) {
@@ -3040,6 +3046,11 @@ tsl::htrie_map<char, field> Collection::get_schema() {
return search_schema;
};
+tsl::htrie_map<char, field> Collection::get_nested_fields() {
+ std::shared_lock lock(mutex);
+ return nested_fields;
+};
+
std::string Collection::get_meta_key(const std::string & collection_name) {
return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;
}
@@ -3078,8 +3089,8 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
return Option<bool>(500, "Error while parsing stored document with sequence ID: " + seq_id_key);
}
- if(!raw_doc) {
- field::flatten_stored_doc(document, search_schema);
+ if(!raw_doc && enable_nested_fields) {
+ field::flatten_stored_doc(document, nested_fields);
}
return Option<bool>(true);
}
@@ -3199,12 +3210,13 @@ Option<bool> Collection::persist_collection_meta() {
Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type,
- const bool do_validation) {
+ const tsl::htrie_map<char, field>& alter_nested_fields) {
// Update schema with additions (deletions can only be made later)
std::vector<field> new_fields;
-
tsl::htrie_map<char, field> schema_additions;
+ std::vector<std::string> nested_field_names;
+
for(auto& f: alter_fields) {
if(f.name == ".*") {
fields.push_back(f);
@@ -3219,6 +3231,15 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
new_fields.push_back(f);
}
+ if(f.nested) {
+ // We should add only root nested fields into `nested_fields` data structure
+ if(alter_nested_fields.count(f.name) != 0) {
+ nested_fields.emplace(f.name, f);
+ }
+
+ nested_field_names.push_back(f.name);
+ }
+
fields.push_back(f);
}
@@ -3248,6 +3269,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
nlohmann::detail::error_handler_t::ignore));
}
+ if(enable_nested_fields) {
+ field::flatten_stored_doc(document, nested_fields);
+ }
+
index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);
iter_batch.emplace_back(std::move(record));
@@ -3265,7 +3290,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions,
- fallback_field_type, token_separators, symbols_to_index, do_validation);
+ fallback_field_type, token_separators, symbols_to_index, true);
iter_batch.clear();
}
@@ -3322,11 +3347,12 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
std::vector<field> del_fields;
std::vector<field> addition_fields;
std::vector<field> reindex_fields;
+ tsl::htrie_map<char, field> new_nested_fields;
std::string this_fallback_field_type;
auto validate_op = validate_alter_payload(alter_payload, addition_fields, reindex_fields,
- del_fields, this_fallback_field_type);
+ new_nested_fields, del_fields, this_fallback_field_type);
if(!validate_op.ok()) {
return validate_op;
}
@@ -3344,14 +3370,14 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
LOG(INFO) << "Processing field additions and deletions first...";
}
- auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, true);
+ auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, new_nested_fields);
if(!batch_alter_op.ok()) {
return batch_alter_op;
}
if(!reindex_fields.empty()) {
LOG(INFO) << "Processing field modifications now...";
- batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, true);
+ batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, new_nested_fields);
if(!batch_alter_op.ok()) {
return batch_alter_op;
}
@@ -3360,17 +3386,11 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
return Option<bool>(true);
}
-void Collection::remove_flat_field_values(nlohmann::json& document) {
+void Collection::remove_flat_fields(nlohmann::json& document) {
if(document.contains(".flat")) {
for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {
document.erase(flat_key);
}
- }
-}
-
-void Collection::remove_flat_fields(nlohmann::json& document) {
- remove_flat_field_values(document);
- if(document.contains(".flat")) {
document.erase(".flat");
}
}
@@ -3459,6 +3479,7 @@ void Collection::prune_doc(nlohmann::json& doc,
Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
std::vector<field>& addition_fields,
std::vector<field>& reindex_fields,
+ tsl::htrie_map<char, field>& new_nested_fields,
std::vector<field>& del_fields,
std::string& fallback_field_type) {
if(!schema_changes.is_object()) {
@@ -3538,6 +3559,23 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
if(found_field) {
del_fields.push_back(field_it.value());
updated_search_schema.erase(field_it.key());
+
+ // handle nested fields
+ if(field_it.value().nested && enable_nested_fields) {
+ std::vector<std::string> sub_fields;
+ auto ex_op = extract_field_name(field_name, search_schema, sub_fields, false, enable_nested_fields);
+ if(!ex_op.ok()) {
+ return ex_op;
+ }
+
+ for(auto& sub_field: sub_fields) {
+ auto sub_field_it = search_schema.find(sub_field);
+ if(sub_field_it != search_schema.end()) {
+ del_fields.push_back(sub_field_it.value());
+ updated_search_schema.erase(sub_field);
+ }
+ }
+ }
}
// NOTE: fields with type "auto" or "string*" will exist in both `search_schema` and `dynamic_fields`
@@ -3584,6 +3622,11 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
} else {
addition_fields.push_back(f);
}
+
+ if(f.nested) {
+ new_nested_fields.emplace(f.name, f);
+ }
+
} else {
// partial update is not supported for now
return Option<bool>(400, "Field `" + field_name + "` is already part of the schema: To "
@@ -3617,19 +3660,27 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
nlohmann::detail::error_handler_t::ignore));
}
- if(!fallback_field_type.empty() || !new_dynamic_fields.empty()) {
+ if(enable_nested_fields) {
+ field::flatten_stored_doc(document, nested_fields);
+ }
+
+ if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !new_nested_fields.empty()) {
std::vector<field> new_fields;
+ std::vector<field> nested_fields_found;
+
Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,
updated_search_schema, new_dynamic_fields,
- nested_fields,
+ new_nested_fields,
fallback_field_type,
- new_fields, enable_nested_fields);
+ new_fields, nested_fields_found,
+ enable_nested_fields);
if(!new_fields_op.ok()) {
return new_fields_op;
}
for(auto& new_field: new_fields) {
updated_search_schema[new_field.name] = new_field;
+ // NOTE: cannot be pushed to `addition_fields` to support drop+re-addition use case
reindex_fields.push_back(new_field);
}
}
@@ -3783,10 +3834,9 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
const tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
std::vector<field>& new_fields,
+ std::vector<field>& nested_fields_found,
const bool enable_nested_fields) {
- std::vector<field> nested_fields_found;
-
if(enable_nested_fields) {
for(auto& nested_field: nested_fields) {
nested_fields_found.push_back(nested_field);
diff --git a/src/field.cpp b/src/field.cpp
index 6d41c877..3c57fda9 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -730,19 +730,9 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
return Option<bool>(true);
}
-Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema) {
- if(document.count(".flat") == 0) {
- return Option<bool>(true);
- }
-
+Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields) {
std::unordered_map<std::string, field> flattened_fields_map;
- for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {
- auto field_it = schema.find(flat_key);
- if(field_it == schema.end()) {
- continue;
- }
-
- const field& nested_field = field_it.value();
+ for(const auto& nested_field: nested_fields) {
std::vector<std::string> field_parts;
StringUtils::split(nested_field.name, field_parts, ".");
@@ -754,6 +744,14 @@ Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htri
flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);
}
+ if(document.count(".flat") == 0) {
+ document[".flat"] = nlohmann::json::array();
+ }
+
+ for(auto& kv: flattened_fields_map) {
+ document[".flat"].push_back(kv.second.name);
+ }
+
return Option<bool>(true);
}
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 09caa17d..e09f57e9 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -293,7 +293,6 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
auto stored_doc = R"({
- ".flat": ["employees.num", "details.name", "details.year"],
"employees": { "num": 1200 },
"foo": "bar",
"details": [{"name": "foo", "year": 2000}]
@@ -312,16 +311,6 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
ASSERT_EQ(1, stored_doc.count("employees.num"));
ASSERT_EQ(1, stored_doc.count("details.name"));
ASSERT_EQ(1, stored_doc.count("details.year"));
-
- // stored doc without any .flat
- stored_doc = R"({
- "employees": { "num": 1200 },
- "foo": "bar",
- "details": [{"name": "foo", "year": 2000}]
- })"_json;
-
- field::flatten_stored_doc(stored_doc, schema);
- ASSERT_EQ(3, stored_doc.size());
}
TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
@@ -384,12 +373,12 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
ASSERT_EQ(0, results["hits"][0].count(".flat"));
ASSERT_EQ(0, results["hits"][0].count("employees.tags"));
- // raw document in the store will have only the .flat meta key but not actual flat fields
+ // raw document in the store will not have the .flat meta key or actual flat fields
nlohmann::json raw_doc;
coll1->get_document_from_store(0, raw_doc, true);
- ASSERT_EQ(1, raw_doc.count(".flat"));
+ ASSERT_EQ(0, raw_doc.count(".flat"));
ASSERT_EQ(0, raw_doc.count("employees.tags"));
- ASSERT_EQ(5, raw_doc.size());
+ ASSERT_EQ(4, raw_doc.size());
ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index eb102cf1..91b65b68 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1197,3 +1197,143 @@ TEST_F(CollectionSchemaChangeTest, DropIntegerFieldAndAddStringValues) {
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
}
+
+TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {
+ // Plain object field
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "person", "type": "object"},
+ {"name": "school.city", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Test";
+ doc["person"] = nlohmann::json::object();
+ doc["person"]["name"] = "Jack";
+ doc["school"] = nlohmann::json::object();
+ doc["school"]["city"] = "NYC";
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ auto fields = coll1->get_fields();
+ auto schema_map = coll1->get_schema();
+
+ ASSERT_EQ(4, fields.size());
+ ASSERT_EQ(4, schema_map.size());
+
+ // drop object field
+
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "person", "drop": true}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ fields = coll1->get_fields();
+ schema_map = coll1->get_schema();
+
+ ASSERT_EQ(2, fields.size());
+ ASSERT_EQ(2, schema_map.size());
+
+ // drop primitive nested field
+
+ schema_changes = R"({
+ "fields": [
+ {"name": "school.city", "drop": true}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ fields = coll1->get_fields();
+ schema_map = coll1->get_schema();
+
+ ASSERT_EQ(1, fields.size());
+ ASSERT_EQ(1, schema_map.size());
+}
+
+TEST_F(CollectionSchemaChangeTest, NestedFieldSchemaAdditions) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Test";
+ doc["person"] = nlohmann::json::object();
+ doc["person"]["name"] = "Jack";
+ doc["school"] = nlohmann::json::object();
+ doc["school"]["city"] = "NYC";
+ doc["school"]["state"] = "NY";
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ auto fields = coll1->get_fields();
+ auto schema_map = coll1->get_schema();
+
+ ASSERT_EQ(1, fields.size());
+ ASSERT_EQ(1, schema_map.size());
+ ASSERT_EQ(0, coll1->get_nested_fields().size());
+
+ // add plain object field
+
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "person", "type": "object"}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ fields = coll1->get_fields();
+ schema_map = coll1->get_schema();
+
+ ASSERT_EQ(3, fields.size());
+ ASSERT_EQ(3, schema_map.size());
+ ASSERT_EQ(1, coll1->get_nested_fields().size());
+
+ // nested primitive field
+
+ schema_changes = R"({
+ "fields": [
+ {"name": "school.city", "type": "string"}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ fields = coll1->get_fields();
+ schema_map = coll1->get_schema();
+
+ ASSERT_EQ(4, fields.size());
+ ASSERT_EQ(4, schema_map.size());
+ ASSERT_EQ(2, coll1->get_nested_fields().size());
+
+ // try searching on new fields
+ auto res_op = coll1->search("jack", {"person.name"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
+
+ res_op = coll1->search("nyc", {"school.city"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(1, res_op.get()["found"].get<size_t>());
+}