Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Don't persist flattened nested field values on disk.
Kishore Nallandff7f8b74603
diff --git a/include/collection.h b/include/collection.hindex b0f09fcc..f0d471c1 100644--- a/include/collection.h+++ b/include/collection.h@@ -216,6 +216,8 @@ private:void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;+ static void remove_flat_field_values(nlohmann::json& document);+static void remove_flat_fields(nlohmann::json& document);bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,@@ -332,9 +334,9 @@ public:static uint32_t get_seq_id_from_key(const std::string & key);- Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;+ Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document, bool raw_doc = false) const;- Option<bool> get_document_from_store(const uint32_t& seq_id, nlohmann::json & document) const;+ Option<bool> get_document_from_store(const uint32_t& seq_id, nlohmann::json & document, bool raw_doc = false) const;Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,const index_operation_t op, const DIRTY_VALUES& dirty_values);diff --git a/include/field.h b/include/field.hindex 40dae976..2f3532b6 100644--- a/include/field.h+++ b/include/field.h@@ -404,6 +404,8 @@ struct field {static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,std::vector<field>& flattened_fields);++ static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema);};struct filter {diff --git a/src/collection.cpp b/src/collection.cppindex 9efd65b9..f5eef56b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -330,6 +330,9 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect}} else {+ // remove flattened field values before storing on disk (.flat meta will be kept)+ remove_flat_field_values(index_record.doc);+const std::string& seq_id_str = std::to_string(index_record.seq_id);const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,nlohmann::detail::error_handler_t::ignore);@@ -3054,24 +3057,13 @@ std::string Collection::get_default_sorting_field() {return default_sorting_field;}-Option<bool> Collection::get_document_from_store(const uint32_t& seq_id, nlohmann::json& document) const {- std::string json_doc_str;- StoreStatus json_doc_status = store->get(get_seq_id_key(seq_id), json_doc_str);-- if(json_doc_status != StoreStatus::FOUND) {- return Option<bool>(500, "Could not locate the JSON document for sequence ID: " + std::to_string(seq_id));- }-- try {- document = nlohmann::json::parse(json_doc_str);- } catch(...) {- return Option<bool>(500, "Error while parsing stored document with sequence ID: " + std::to_string(seq_id));- }-- return Option<bool>(true);+Option<bool> Collection::get_document_from_store(const uint32_t& seq_id,+ nlohmann::json& document, bool raw_doc) const {+ return get_document_from_store(get_seq_id_key(seq_id), document, raw_doc);}-Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) const {+Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,+ nlohmann::json& document, bool raw_doc) const {std::string json_doc_str;StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);@@ -3086,6 +3078,9 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,return Option<bool>(500, "Error while parsing stored document with sequence ID: " + seq_id_key);}+ if(!raw_doc) {+ field::flatten_stored_doc(document, search_schema);+ }return Option<bool>(true);}@@ -3365,11 +3360,17 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {return Option<bool>(true);}-void Collection::remove_flat_fields(nlohmann::json& document) {+void Collection::remove_flat_field_values(nlohmann::json& document) {if(document.contains(".flat")) {for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {document.erase(flat_key);}+ }+}++void Collection::remove_flat_fields(nlohmann::json& document) {+ remove_flat_field_values(document);+ if(document.contains(".flat")) {document.erase(".flat");}}diff --git a/src/field.cpp b/src/field.cppindex e67bcc80..6d41c877 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -729,3 +729,31 @@ Option<bool> field::flatten_doc(nlohmann::json& document,return Option<bool>(true);}++Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& schema) {+ if(document.count(".flat") == 0) {+ return Option<bool>(true);+ }++ std::unordered_map<std::string, field> flattened_fields_map;+ for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {+ auto field_it = schema.find(flat_key);+ if(field_it == schema.end()) {+ continue;+ }++ const field& nested_field = field_it.value();+ std::vector<std::string> field_parts;+ StringUtils::split(nested_field.name, field_parts, ".");++ if(field_parts.size() > 1 && document.count(nested_field.name) != 0) {+ // skip explicitly present nested fields+ continue;+ }++ flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);+ }++ return Option<bool>(true);+}+diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 8112192c..09caa17d 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -291,6 +291,39 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());}+TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {+ auto stored_doc = R"({+ ".flat": ["employees.num", "details.name", "details.year"],+ "employees": { "num": 1200 },+ "foo": "bar",+ "details": [{"name": "foo", "year": 2000}]+ })"_json;++ tsl::htrie_map<char, field> schema;+ schema.emplace("employees.num", field("employees.num", field_types::INT32, false));+ schema.emplace("details.name", field("details.name", field_types::STRING_ARRAY, false));+ schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));++ field::flatten_stored_doc(stored_doc, schema);++ ASSERT_EQ(3, stored_doc[".flat"].size());+ ASSERT_EQ(7, stored_doc.size());++ ASSERT_EQ(1, stored_doc.count("employees.num"));+ ASSERT_EQ(1, stored_doc.count("details.name"));+ ASSERT_EQ(1, stored_doc.count("details.year"));++ // stored doc without any .flat+ stored_doc = R"({+ "employees": { "num": 1200 },+ "foo": "bar",+ "details": [{"name": "foo", "year": 2000}]+ })"_json;++ field::flatten_stored_doc(stored_doc, schema);+ ASSERT_EQ(3, stored_doc.size());+}+TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};@@ -347,6 +380,17 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {]})"_json;+ // ensure that flat fields are not returned in response+ ASSERT_EQ(0, results["hits"][0].count(".flat"));+ ASSERT_EQ(0, results["hits"][0].count("employees.tags"));++ // raw document in the store will have only the .flat meta key but not actual flat fields+ nlohmann::json raw_doc;+ coll1->get_document_from_store(0, raw_doc, true);+ ASSERT_EQ(1, raw_doc.count(".flat"));+ ASSERT_EQ(0, raw_doc.count("employees.tags"));+ ASSERT_EQ(5, raw_doc.size());+ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());