Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix include/exclude fields in export for nested docs.
Kishore Nallan502e7503b31d
diff --git a/include/collection.h b/include/collection.hindex 171b1457..94669193 100644--- a/include/collection.h+++ b/include/collection.h@@ -215,8 +215,6 @@ private:void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;- static void remove_flat_fields(nlohmann::json& document);-bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,@@ -251,6 +249,11 @@ private:static uint64_t extract_bits(uint64_t value, unsigned lsb_offset, unsigned n);+ Option<bool> populate_include_exclude_fields(const spp::sparse_hash_set<std::string>& include_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ tsl::htrie_set<char>& include_fields_full,+ tsl::htrie_set<char>& exclude_fields_full) const;+public:enum {MAX_ARRAY_MATCHES = 5};@@ -337,6 +340,8 @@ public:Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,const index_operation_t op, const DIRTY_VALUES& dirty_values);+ static void remove_flat_fields(nlohmann::json& document);+static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0);@@ -372,6 +377,11 @@ public:const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT,const bool& return_doc=false, const bool& return_id=false);+ Option<bool> populate_include_exclude_fields_lk(const spp::sparse_hash_set<std::string>& include_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ tsl::htrie_set<char>& include_fields_full,+ tsl::htrie_set<char>& exclude_fields_full) const;+Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,diff --git a/include/core_api_utils.h b/include/core_api_utils.hindex 4e057035..cc266576 100644--- a/include/core_api_utils.h+++ b/include/core_api_utils.h@@ -22,8 +22,8 @@ struct export_state_t: public req_state_t {Collection* collection;std::vector<std::pair<size_t, uint32_t*>> index_ids;std::vector<size_t> offsets;- std::set<std::string> include_fields;- std::set<std::string> exclude_fields;+ tsl::htrie_set<char> include_fields;+ tsl::htrie_set<char> exclude_fields;size_t export_batch_size = 100;std::string* res_body;diff --git a/src/collection.cpp b/src/collection.cppindex 0ea060fa..4b874fc5 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1012,49 +1012,16 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}- std::vector<std::string> include_fields_vec;- std::vector<std::string> exclude_fields_vec;tsl::htrie_set<char> include_fields_full;tsl::htrie_set<char> exclude_fields_full;- for(auto& f_name: include_fields) {- auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields);- if(!field_op.ok()) {- if(field_op.code() == 404) {- // field need not be part of schema to be included (could be a stored value in the doc)- include_fields_vec.push_back(f_name);- continue;- }- return Option<nlohmann::json>(field_op.code(), field_op.error());- }- }-- for(auto& f_name: exclude_fields) {- if(f_name == "out_of") {- // `out_of` is strictly a meta-field, but we handle it since it's useful- continue;- }-- auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields);- if(!field_op.ok()) {- if(field_op.code() == 404) {- // field need not be part of schema to be excluded (could be a stored value in the doc)- exclude_fields_vec.push_back(f_name);- continue;- }- return Option<nlohmann::json>(field_op.code(), field_op.error());- }- }-- for(auto& f_name: include_fields_vec) {- include_fields_full.insert(f_name);- }+ auto include_exclude_op = populate_include_exclude_fields(include_fields, exclude_fields,+ include_fields_full, exclude_fields_full);- for(auto& f_name: exclude_fields_vec) {- exclude_fields_full.insert(f_name);+ if(!include_exclude_op.ok()) {+ return Option<nlohmann::json>(include_exclude_op.code(), include_exclude_op.error());}-// process weights for search fieldsstd::vector<std::string> reordered_search_fields;std::vector<search_field_t> weighted_search_fields;@@ -4111,4 +4078,60 @@ std::string Collection::get_fallback_field_type() {bool Collection::get_enable_nested_fields() {return enable_nested_fields;-};\ No newline at end of file+};++Option<bool> Collection::populate_include_exclude_fields(const spp::sparse_hash_set<std::string>& include_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ tsl::htrie_set<char>& include_fields_full,+ tsl::htrie_set<char>& exclude_fields_full) const {++ std::vector<std::string> include_fields_vec;+ std::vector<std::string> exclude_fields_vec;++ for(auto& f_name: include_fields) {+ auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields);+ if(!field_op.ok()) {+ if(field_op.code() == 404) {+ // field need not be part of schema to be included (could be a stored value in the doc)+ include_fields_vec.push_back(f_name);+ continue;+ }+ return Option<bool>(field_op.code(), field_op.error());+ }+ }++ for(auto& f_name: exclude_fields) {+ if(f_name == "out_of") {+ // `out_of` is strictly a meta-field, but we handle it since it's useful+ continue;+ }++ auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields);+ if(!field_op.ok()) {+ if(field_op.code() == 404) {+ // field need not be part of schema to be excluded (could be a stored value in the doc)+ exclude_fields_vec.push_back(f_name);+ continue;+ }+ return Option<bool>(field_op.code(), field_op.error());+ }+ }++ for(auto& f_name: include_fields_vec) {+ include_fields_full.insert(f_name);+ }++ for(auto& f_name: exclude_fields_vec) {+ exclude_fields_full.insert(f_name);+ }++ return Option<bool>(true);+}++Option<bool> Collection::populate_include_exclude_fields_lk(const spp::sparse_hash_set<std::string>& include_fields,+ const spp::sparse_hash_set<std::string>& exclude_fields,+ tsl::htrie_set<char>& include_fields_full,+ tsl::htrie_set<char>& exclude_fields_full) const {+ std::shared_lock lock(mutex);+ return populate_include_exclude_fields(include_fields, exclude_fields, include_fields_full, exclude_fields_full);+}diff --git a/src/core_api.cpp b/src/core_api.cppindex 6e76e97b..846f0b6c 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -603,6 +603,8 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::sharereq->data = export_state;std::string simple_filter_query;+ spp::sparse_hash_set<std::string> exclude_fields;+ spp::sparse_hash_set<std::string> include_fields;if(req->params.count(FILTER_BY) != 0) {simple_filter_query = req->params[FILTER_BY];@@ -611,15 +613,18 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::shareif(req->params.count(INCLUDE_FIELDS) != 0) {std::vector<std::string> include_fields_vec;StringUtils::split(req->params[INCLUDE_FIELDS], include_fields_vec, ",");- export_state->include_fields = std::set<std::string>(include_fields_vec.begin(), include_fields_vec.end());+ include_fields = spp::sparse_hash_set<std::string>(include_fields_vec.begin(), include_fields_vec.end());}if(req->params.count(EXCLUDE_FIELDS) != 0) {std::vector<std::string> exclude_fields_vec;StringUtils::split(req->params[EXCLUDE_FIELDS], exclude_fields_vec, ",");- export_state->exclude_fields = std::set<std::string>(exclude_fields_vec.begin(), exclude_fields_vec.end());+ exclude_fields = spp::sparse_hash_set<std::string>(exclude_fields_vec.begin(), exclude_fields_vec.end());}+ collection->populate_include_exclude_fields_lk(include_fields, exclude_fields,+ export_state->include_fields, export_state->exclude_fields);+if(req->params.count(BATCH_SIZE) != 0 && StringUtils::is_uint32_t(req->params[BATCH_SIZE])) {export_state->export_batch_size = std::stoul(req->params[BATCH_SIZE]);}@@ -659,20 +664,8 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::shareres->body += it->value().ToString();} else {nlohmann::json doc = nlohmann::json::parse(it->value().ToString());- nlohmann::json filtered_doc;- for(const auto& kv: doc.items()) {- bool must_include = export_state->include_fields.empty() ||- (export_state->include_fields.count(kv.key()) != 0);-- bool must_exclude = !export_state->exclude_fields.empty() &&- (export_state->exclude_fields.count(kv.key()) != 0);-- if(must_include && !must_exclude) {- filtered_doc[kv.key()] = kv.value();- }- }-- res->body += filtered_doc.dump();+ Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);+ res->body += doc.dump();}it->Next();diff --git a/src/core_api_utils.cpp b/src/core_api_utils.cppindex 7c88adec..fda1777b 100644--- a/src/core_api_utils.cpp+++ b/src/core_api_utils.cpp@@ -66,20 +66,9 @@ Option<bool> stateful_export_docs(export_state_t* export_state, size_t batch_sizif(export_state->include_fields.empty() && export_state->exclude_fields.empty()) {export_state->res_body->append(doc.dump());} else {- nlohmann::json filtered_doc;- for(const auto& kv: doc.items()) {- bool must_include = export_state->include_fields.empty() ||- (export_state->include_fields.count(kv.key()) != 0);-- bool must_exclude = !export_state->exclude_fields.empty() &&- (export_state->exclude_fields.count(kv.key()) != 0);-- if(must_include && !must_exclude) {- filtered_doc[kv.key()] = kv.value();- }- }-- export_state->res_body->append(filtered_doc.dump());+ Collection::remove_flat_fields(doc);+ Collection::prune_doc(doc, export_state->include_fields, export_state->exclude_fields);+ export_state->res_body->append(doc.dump());}export_state->res_body->append("\n");diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex d8d8787b..2545fe13 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -559,3 +559,124 @@ TEST_F(CoreAPIUtilsTest, ExportWithFilter) {ASSERT_TRUE(done);ASSERT_EQ('}', export_state.res_body->back());}++TEST_F(CoreAPIUtilsTest, ExportIncludeExcludeFields) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "name", "type": "object" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "name": {"first": "John", "last": "Smith"},+ "points": 100+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req->params["collection"] = "coll1";++ // include fields++ req->params["include_fields"] = "name.last";++ get_export_documents(req, res);++ std::vector<std::string> res_strs;+ StringUtils::split(res->body, res_strs, "\n");+ nlohmann::json doc = nlohmann::json::parse(res_strs[0]);+ ASSERT_EQ(1, doc.size());+ ASSERT_EQ(1, doc.count("name"));+ ASSERT_EQ(1, doc["name"].count("last"));++ // exclude fields++ delete dynamic_cast<deletion_state_t*>(req->data);+ req->data = nullptr;+ res->body.clear();+ req->params.erase("include_fields");+ req->params["exclude_fields"] = "name.last";+ get_export_documents(req, res);++ res_strs.clear();+ StringUtils::split(res->body, res_strs, "\n");+ doc = nlohmann::json::parse(res_strs[0]);+ ASSERT_EQ(3, doc.size());+ ASSERT_EQ(1, doc.count("id"));+ ASSERT_EQ(1, doc.count("points"));+ ASSERT_EQ(1, doc.count("name"));+ ASSERT_EQ(1, doc["name"].count("first"));++ collectionManager.drop_collection("coll1");+}++TEST_F(CoreAPIUtilsTest, ExportIncludeExcludeFieldsWithFilter) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "name", "type": "object" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "name": {"first": "John", "last": "Smith"},+ "points": 100+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req->params["collection"] = "coll1";++ // include fields++ req->params["include_fields"] = "name.last";+ req->params["filter_by"] = "points:>=0";++ get_export_documents(req, res);++ std::vector<std::string> res_strs;+ StringUtils::split(res->body, res_strs, "\n");+ nlohmann::json doc = nlohmann::json::parse(res_strs[0]);+ ASSERT_EQ(1, doc.size());+ ASSERT_EQ(1, doc.count("name"));+ ASSERT_EQ(1, doc["name"].count("last"));++ // exclude fields++ delete dynamic_cast<deletion_state_t*>(req->data);+ req->data = nullptr;+ res->body.clear();+ req->params.erase("include_fields");+ req->params["exclude_fields"] = "name.last";+ get_export_documents(req, res);++ res_strs.clear();+ StringUtils::split(res->body, res_strs, "\n");+ doc = nlohmann::json::parse(res_strs[0]);+ ASSERT_EQ(3, doc.size());+ ASSERT_EQ(1, doc.count("id"));+ ASSERT_EQ(1, doc.count("points"));+ ASSERT_EQ(1, doc.count("name"));+ ASSERT_EQ(1, doc["name"].count("first"));++ collectionManager.drop_collection("coll1");+}\ No newline at end of file