Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Don't persist flattened nested field values on disk.

Kishore Nallandff7f8b74603
diff --git a/include/collection.h b/include/collection.h
index b0f09fcc..f0d471c1 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -216,6 +216,8 @@ private:
void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;
+ static void remove_flat_field_values(nlohmann::json& document);
+
static void remove_flat_fields(nlohmann::json& document);
bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,
@@ -332,9 +334,9 @@ public:
static uint32_t get_seq_id_from_key(const std::string & key);
- Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;
+ Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document, bool raw_doc = false) const;
- Option<bool> get_document_from_store(const uint32_t& seq_id, nlohmann::json & document) const;
+ Option<bool> get_document_from_store(const uint32_t& seq_id, nlohmann::json & document, bool raw_doc = false) const;
Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,
const index_operation_t op, const DIRTY_VALUES& dirty_values);
diff --git a/include/field.h b/include/field.h
index 40dae976..2f3532b6 100644
--- a/include/field.h
+++ b/include/field.h
@@ -404,6 +404,8 @@ struct field {
static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,
std::vector<field>& flattened_fields);
+
+ static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema);
};
struct filter {
diff --git a/src/collection.cpp b/src/collection.cpp
index 9efd65b9..f5eef56b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -330,6 +330,9 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect
}
} else {
+ // remove flattened field values before storing on disk (.flat meta will be kept)
+ remove_flat_field_values(index_record.doc);
+
const std::string& seq_id_str = std::to_string(index_record.seq_id);
const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,
nlohmann::detail::error_handler_t::ignore);
@@ -3054,24 +3057,13 @@ std::string Collection::get_default_sorting_field() {
return default_sorting_field;
}
-Option<bool> Collection::get_document_from_store(const uint32_t& seq_id, nlohmann::json& document) const {
- std::string json_doc_str;
- StoreStatus json_doc_status = store->get(get_seq_id_key(seq_id), json_doc_str);
-
- if(json_doc_status != StoreStatus::FOUND) {
- return Option<bool>(500, "Could not locate the JSON document for sequence ID: " + std::to_string(seq_id));
- }
-
- try {
- document = nlohmann::json::parse(json_doc_str);
- } catch(...) {
- return Option<bool>(500, "Error while parsing stored document with sequence ID: " + std::to_string(seq_id));
- }
-
- return Option<bool>(true);
+Option<bool> Collection::get_document_from_store(const uint32_t& seq_id,
+ nlohmann::json& document, bool raw_doc) const {
+ return get_document_from_store(get_seq_id_key(seq_id), document, raw_doc);
}
-Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) const {
+Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
+ nlohmann::json& document, bool raw_doc) const {
std::string json_doc_str;
StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
@@ -3086,6 +3078,9 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
return Option<bool>(500, "Error while parsing stored document with sequence ID: " + seq_id_key);
}
+ if(!raw_doc) {
+ field::flatten_stored_doc(document, search_schema);
+ }
return Option<bool>(true);
}
@@ -3365,11 +3360,17 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
return Option<bool>(true);
}
-void Collection::remove_flat_fields(nlohmann::json& document) {
+void Collection::remove_flat_field_values(nlohmann::json& document) {
if(document.contains(".flat")) {
for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {
document.erase(flat_key);
}
+ }
+}
+
+void Collection::remove_flat_fields(nlohmann::json& document) {
+ remove_flat_field_values(document);
+ if(document.contains(".flat")) {
document.erase(".flat");
}
}
diff --git a/src/field.cpp b/src/field.cpp
index e67bcc80..6d41c877 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -729,3 +729,31 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
return Option<bool>(true);
}
+
+Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema) {
+ if(document.count(".flat") == 0) {
+ return Option<bool>(true);
+ }
+
+ std::unordered_map<std::string, field> flattened_fields_map;
+ for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {
+ auto field_it = schema.find(flat_key);
+ if(field_it == schema.end()) {
+ continue;
+ }
+
+ const field& nested_field = field_it.value();
+ std::vector<std::string> field_parts;
+ StringUtils::split(nested_field.name, field_parts, ".");
+
+ if(field_parts.size() > 1 && document.count(nested_field.name) != 0) {
+ // skip explicitly present nested fields
+ continue;
+ }
+
+ flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);
+ }
+
+ return Option<bool>(true);
+}
+
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 8112192c..09caa17d 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -291,6 +291,39 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());
}
+TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
+ auto stored_doc = R"({
+ ".flat": ["employees.num", "details.name", "details.year"],
+ "employees": { "num": 1200 },
+ "foo": "bar",
+ "details": [{"name": "foo", "year": 2000}]
+ })"_json;
+
+ tsl::htrie_map<char, field> schema;
+ schema.emplace("employees.num", field("employees.num", field_types::INT32, false));
+ schema.emplace("details.name", field("details.name", field_types::STRING_ARRAY, false));
+ schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));
+
+ field::flatten_stored_doc(stored_doc, schema);
+
+ ASSERT_EQ(3, stored_doc[".flat"].size());
+ ASSERT_EQ(7, stored_doc.size());
+
+ ASSERT_EQ(1, stored_doc.count("employees.num"));
+ ASSERT_EQ(1, stored_doc.count("details.name"));
+ ASSERT_EQ(1, stored_doc.count("details.year"));
+
+ // stored doc without any .flat
+ stored_doc = R"({
+ "employees": { "num": 1200 },
+ "foo": "bar",
+ "details": [{"name": "foo", "year": 2000}]
+ })"_json;
+
+ field::flatten_stored_doc(stored_doc, schema);
+ ASSERT_EQ(3, stored_doc.size());
+}
+
TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};
@@ -347,6 +380,17 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
]
})"_json;
+ // ensure that flat fields are not returned in response
+ ASSERT_EQ(0, results["hits"][0].count(".flat"));
+ ASSERT_EQ(0, results["hits"][0].count("employees.tags"));
+
+ // raw document in the store will have only the .flat meta key but not actual flat fields
+ nlohmann::json raw_doc;
+ coll1->get_document_from_store(0, raw_doc, true);
+ ASSERT_EQ(1, raw_doc.count(".flat"));
+ ASSERT_EQ(0, raw_doc.count("employees.tags"));
+ ASSERT_EQ(5, raw_doc.size());
+
ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());