Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Reference `include_fields`.
Harpreet Sangar7a73d9ffd0f4
diff --git a/include/collection.h b/include/collection.hindex 72ef1280..e6f54e3e 100644--- a/include/collection.h+++ b/include/collection.h@@ -356,8 +356,10 @@ public:static void remove_flat_fields(nlohmann::json& document);- static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,- const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0);+ static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0,+ const uint32_t doc_sequence_id = 0, const std::string& collection_name = "",+ const std::map<std::string, std::string>& reference_filter_map = {});const Index* _get_index() const;@@ -449,6 +451,8 @@ public:Option<bool> get_filter_ids(const std::string & filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids) const;+ Option<std::string> get_reference_field(const std::string & collection_name) const;+Option<bool> get_reference_filter_ids(const std::string & filter_query,const std::string & collection_name,std::pair<uint32_t, uint32_t*>& reference_index_ids) const;@@ -515,8 +519,8 @@ public:void process_highlight_fields(const std::vector<search_field_t>& search_fields,const std::vector<std::string>& raw_search_fields,- const tsl::htrie_set<char>& exclude_fields,const tsl::htrie_set<char>& include_fields,+ const tsl::htrie_set<char>& exclude_fields,const std::vector<std::string>& highlight_field_names,const std::vector<std::string>& highlight_full_field_names,const std::vector<enable_t>& infixes,diff --git a/include/string_utils.h b/include/string_utils.hindex 3ba28925..5ad2c9a6 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -379,4 +379,6 @@ struct StringUtils {static size_t get_num_chars(const std::string& text);static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);++ static Option<bool> split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens);};diff --git a/src/collection.cpp b/src/collection.cppindex 5181f690..97c8072b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -945,6 +945,12 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,const bool extract_only_string_fields,const bool enable_nested_fields,const bool handle_wildcard) {+ // Reference to other collection+ if (field_name[0] == '$') {+ processed_search_fields.push_back(field_name);+ return Option<bool>(true);+ }+if(field_name == "id") {processed_search_fields.push_back(field_name);return Option<bool>(true);@@ -990,6 +996,19 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,return Option<bool>(true);}+void get_reference_filters(filter_node_t const* const root, std::map<std::string, std::string>& reference_filter_map) {+ if (root == nullptr) {+ return;+ }++ if (!root->isOperator && !root->filter_exp.referenced_collection_name.empty()) {+ reference_filter_map[root->filter_exp.referenced_collection_name] = root->filter_exp.field_name;+ }++ get_reference_filters(root->left, reference_filter_map);+ get_reference_filters(root->right, reference_filter_map);+}+Option<nlohmann::json> Collection::search(const std::string & raw_query,const std::vector<std::string>& raw_search_fields,const std::string & filter_query, const std::vector<std::string>& facet_fields,@@ -1753,7 +1772,24 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}remove_flat_fields(document);- prune_doc(document, include_fields_full, exclude_fields_full);+ auto doc_id_op = doc_id_to_seq_id(document["id"].get<std::string>());+ if (!doc_id_op.ok()) {+ return Option<nlohmann::json>(doc_id_op.code(), doc_id_op.error());+ }++ std::map<std::string, std::string> reference_filter_map;+ get_reference_filters(filter_tree_root, reference_filter_map);+ auto prune_op = prune_doc(document,+ include_fields_full,+ exclude_fields_full,+ "",+ 0,+ doc_id_op.get(),+ name,+ reference_filter_map);+ if (!prune_op.ok()) {+ return Option<nlohmann::json>(prune_op.code(), prune_op.error());+ }wrapper_doc["document"] = document;wrapper_doc["highlight"] = highlight_res;@@ -2418,9 +2454,7 @@ Option<bool> Collection::get_filter_ids(const std::string & filter_query,return Option<bool>(true);}-Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,- const std::string & collection_name,- std::pair<uint32_t, uint32_t*>& reference_index_ids) const {+Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {std::shared_lock lock(mutex);std::string reference_field_name;@@ -2433,10 +2467,23 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que}if (reference_field_name.empty()) {- return Option<bool>(400, "Could not find any field in `" + name + "` referencing the collection `"- + collection_name + "`.");+ return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"+ + collection_name + "`.");}+ return Option(reference_field_name);+}++Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,+ const std::string & collection_name,+ std::pair<uint32_t, uint32_t*>& reference_index_ids) const {+ auto reference_field_op = get_reference_field(collection_name);+ if (!reference_field_op.ok()) {+ return Option<bool>(reference_field_op.code(), reference_field_op.error());+ }++ std::shared_lock lock(mutex);+const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";filter_node_t* filter_tree_root = nullptr;Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,@@ -2446,8 +2493,8 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que}// Reference helper field has the sequence id of other collection's documents.- reference_field_name += REFERENCE_HELPER_FIELD_SUFFIX;- index->do_reference_filtering_with_lock(reference_index_ids, filter_tree_root, reference_field_name);+ auto field_name = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;+ index->do_reference_filtering_with_lock(reference_index_ids, filter_tree_root, field_name);delete filter_tree_root;return Option<bool>(true);@@ -3690,10 +3737,12 @@ void Collection::remove_flat_fields(nlohmann::json& document) {}}-void Collection::prune_doc(nlohmann::json& doc,+Option<bool> Collection::prune_doc(nlohmann::json& doc,const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,- const std::string& parent_name, size_t depth) {+ const std::string& parent_name, size_t depth,+ const uint32_t doc_sequence_id, const std::string& collection_name,+ const std::map<std::string, std::string>& reference_filter_map) {// doc can only be an objectauto it = doc.begin();while(it != doc.end()) {@@ -3768,6 +3817,79 @@ void Collection::prune_doc(nlohmann::json& doc,it++;}++ auto reference_it = include_names.equal_prefix_range("$");+ for (auto reference = reference_it.first; reference != reference_it.second; reference++) {+ auto ref = reference.key();+ size_t parenthesis_index = ref.find('(');++ auto ref_collection_name = ref.substr(1, parenthesis_index - 1);+ auto reference_fields = ref.substr(parenthesis_index + 1, ref.size() - parenthesis_index - 2);++ auto& cm = CollectionManager::get_instance();+ auto collection = cm.get_collection(ref_collection_name);+ if (collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` not found.");+ }++ std::vector<std::string> include_fields_vec;+ StringUtils::split(reference_fields, include_fields_vec, ",");++ spp::sparse_hash_set<std::string> include_fields, exclude_fields;+ include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());++ tsl::htrie_set<char> include_fields_full, exclude_fields_full;+ auto include_exclude_op = collection->populate_include_exclude_fields(include_fields, exclude_fields,+ include_fields_full, exclude_fields_full);+ if (!include_exclude_op.ok()) {+ return include_exclude_op;+ }++ auto reference_field_op = collection->get_reference_field(collection_name);+ if (!reference_field_op.ok()) {+ return Option<bool>(reference_field_op.code(), reference_field_op.error());+ }++ std::vector<std::pair<size_t, uint32_t*>> documents;+ auto filter = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX + ":=" + std::to_string(doc_sequence_id);+ if (reference_filter_map.count(ref_collection_name) > 0) {+ filter += "&&";+ filter += reference_filter_map.at(ref_collection_name);+ }+ auto filter_op = collection->get_filter_ids(filter, documents);+ if (!filter_op.ok()) {+ return filter_op;+ }++ if (documents[0].first == 0) {+ continue;+ }++ std::vector<nlohmann::json> reference_docs;+ reference_docs.reserve(documents[0].first);+ for (size_t i = 0; i < documents[0].first; i++) {+ auto doc_seq_id = documents[0].second[i];++ nlohmann::json ref_doc;+ auto get_doc_op = collection->get_document_from_store(doc_seq_id, ref_doc);+ if (!get_doc_op.ok()) {+ return get_doc_op;+ }++ auto prune_op = prune_doc(ref_doc, include_fields_full, exclude_fields_full);+ if (!prune_op.ok()) {+ return prune_op;+ }++ reference_docs.push_back(ref_doc);+ }++ for (const auto &ref_doc: reference_docs) {+ doc.update(ref_doc);+ }+ }++ return Option<bool>(true);}Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 01853fd3..97a7954b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -913,6 +913,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reif(key == FACET_BY){StringUtils::split_facet(val, *find_str_list_it->second);}+ else if(key == INCLUDE_FIELDS){+ auto op = StringUtils::split_include_fields(val, *find_str_list_it->second);+ if (!op.ok()) {+ return op;+ }+ }else{StringUtils::split(val, *find_str_list_it->second, ",");}diff --git a/src/string_utils.cpp b/src/string_utils.cppindex a9409400..9fa65b4c 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -453,6 +453,42 @@ Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query,return Option<bool>(true);}+Option<bool> StringUtils::split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens) {+ size_t start = 0, end = 0, size = include_fields.size();+ std::string include_field;++ while (true) {+ auto range_pos = include_fields.find('$', start);+ auto comma_pos = include_fields.find(',', start);++ if (range_pos == std::string::npos && comma_pos == std::string::npos) {+ if (start < size - 1) {+ tokens.push_back(include_fields.substr(start, size - start));+ }+ break;+ } else if (range_pos < comma_pos) {+ end = include_fields.find(')', range_pos);+ if (end == std::string::npos || end < include_fields.find('(', range_pos)) {+ return Option<bool>(400, "Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.");+ }++ include_field = include_fields.substr(range_pos, (end - range_pos) + 1);+ } else {+ end = comma_pos;+ include_field = include_fields.substr(start, end - start);+ }++ include_field = trim(include_field);+ if (!include_field.empty()) {+ tokens.push_back(include_field);+ }++ start = end + 1;+ }++ return Option<bool>(true);+}+/*size_t StringUtils::unicode_length(const std::string& bytes) {std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;return utf8conv.from_bytes(bytes).size();diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 98e7663f..e96f7f57 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -549,4 +549,164 @@ TEST_F(CollectionJoinTest, FilterByReference_MultipleMatch) {collectionManager.drop_collection("Users");collectionManager.drop_collection("Repos");collectionManager.drop_collection("Links");+}++TEST_F(CollectionJoinTest, IncludeFieldsByReference_SingleMatch) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string"},+ {"name": "product_description", "type": "string"}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."+ })"_json,+ R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."+ })"_json+ };+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Customers",+ "fields": [+ {"name": "customer_id", "type": "string"},+ {"name": "customer_name", "type": "string"},+ {"name": "product_price", "type": "float"},+ {"name": "product_id", "type": "string", "reference": "Products.product_id"}+ ]+ })"_json;+ documents = {+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 143,+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 73.5,+ "product_id": "product_b"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 75,+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 140,+ "product_id": "product_b"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ std::map<std::string, std::string> req_params = {+ {"collection", "Products"},+ {"q", "s"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ };++ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ req_params["include_fields"] = "$foo.bar";+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());++ req_params["include_fields"] = "$foo(bar";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());++ req_params["include_fields"] = "$foo(bar)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Referenced collection `foo` not found.", search_op.error());++ req_params["include_fields"] = "$Customers(bar)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ nlohmann::json res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(0, res_obj["hits"][0]["document"].size());++ req_params["include_fields"] = "$Customers(product_price)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));++ req_params["include_fields"] = "$Customers(product_price, customer_id)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));+ ASSERT_EQ("customer_a", res_obj["hits"][0]["document"].at("customer_id"));++ req_params["include_fields"] = "*, $Customers(product_price, customer_id)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ // 3 fields in Products document and 2 fields from Customers document+ ASSERT_EQ(5, res_obj["hits"][0]["document"].size());++ req_params["include_fields"] = "*, $Customers(product*)";+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ // 3 fields in Products document and 2 fields from Customers document+ ASSERT_EQ(5, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));}\ No newline at end of file