Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Reference `include_fields`.

Harpreet Sangar7a73d9ffd0f4
diff --git a/include/collection.h b/include/collection.h
index 72ef1280..e6f54e3e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -356,8 +356,10 @@ public:
static void remove_flat_fields(nlohmann::json& document);
- static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
- const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0);
+ static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0,
+ const uint32_t doc_sequence_id = 0, const std::string& collection_name = "",
+ const std::map<std::string, std::string>& reference_filter_map = {});
const Index* _get_index() const;
@@ -449,6 +451,8 @@ public:
Option<bool> get_filter_ids(const std::string & filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids) const;
+ Option<std::string> get_reference_field(const std::string & collection_name) const;
+
Option<bool> get_reference_filter_ids(const std::string & filter_query,
const std::string & collection_name,
std::pair<uint32_t, uint32_t*>& reference_index_ids) const;
@@ -515,8 +519,8 @@ public:
void process_highlight_fields(const std::vector<search_field_t>& search_fields,
const std::vector<std::string>& raw_search_fields,
- const tsl::htrie_set<char>& exclude_fields,
const tsl::htrie_set<char>& include_fields,
+ const tsl::htrie_set<char>& exclude_fields,
const std::vector<std::string>& highlight_field_names,
const std::vector<std::string>& highlight_full_field_names,
const std::vector<enable_t>& infixes,
diff --git a/include/string_utils.h b/include/string_utils.h
index 3ba28925..5ad2c9a6 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -379,4 +379,6 @@ struct StringUtils {
static size_t get_num_chars(const std::string& text);
static Option<bool> tokenize_filter_query(const std::string& filter_query, std::queue<std::string>& tokens);
+
+ static Option<bool> split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 5181f690..97c8072b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -945,6 +945,12 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
const bool extract_only_string_fields,
const bool enable_nested_fields,
const bool handle_wildcard) {
+ // Reference to other collection
+ if (field_name[0] == '$') {
+ processed_search_fields.push_back(field_name);
+ return Option<bool>(true);
+ }
+
if(field_name == "id") {
processed_search_fields.push_back(field_name);
return Option<bool>(true);
@@ -990,6 +996,19 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
return Option<bool>(true);
}
+void get_reference_filters(filter_node_t const* const root, std::map<std::string, std::string>& reference_filter_map) {
+ if (root == nullptr) {
+ return;
+ }
+
+ if (!root->isOperator && !root->filter_exp.referenced_collection_name.empty()) {
+ reference_filter_map[root->filter_exp.referenced_collection_name] = root->filter_exp.field_name;
+ }
+
+ get_reference_filters(root->left, reference_filter_map);
+ get_reference_filters(root->right, reference_filter_map);
+}
+
Option<nlohmann::json> Collection::search(const std::string & raw_query,
const std::vector<std::string>& raw_search_fields,
const std::string & filter_query, const std::vector<std::string>& facet_fields,
@@ -1753,7 +1772,24 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
remove_flat_fields(document);
- prune_doc(document, include_fields_full, exclude_fields_full);
+ auto doc_id_op = doc_id_to_seq_id(document["id"].get<std::string>());
+ if (!doc_id_op.ok()) {
+ return Option<nlohmann::json>(doc_id_op.code(), doc_id_op.error());
+ }
+
+ std::map<std::string, std::string> reference_filter_map;
+ get_reference_filters(filter_tree_root, reference_filter_map);
+ auto prune_op = prune_doc(document,
+ include_fields_full,
+ exclude_fields_full,
+ "",
+ 0,
+ doc_id_op.get(),
+ name,
+ reference_filter_map);
+ if (!prune_op.ok()) {
+ return Option<nlohmann::json>(prune_op.code(), prune_op.error());
+ }
wrapper_doc["document"] = document;
wrapper_doc["highlight"] = highlight_res;
@@ -2418,9 +2454,7 @@ Option<bool> Collection::get_filter_ids(const std::string & filter_query,
return Option<bool>(true);
}
-Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,
- const std::string & collection_name,
- std::pair<uint32_t, uint32_t*>& reference_index_ids) const {
+Option<std::string> Collection::get_reference_field(const std::string & collection_name) const {
std::shared_lock lock(mutex);
std::string reference_field_name;
@@ -2433,10 +2467,23 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que
}
if (reference_field_name.empty()) {
- return Option<bool>(400, "Could not find any field in `" + name + "` referencing the collection `"
- + collection_name + "`.");
+ return Option<std::string>(400, "Could not find any field in `" + name + "` referencing the collection `"
+ + collection_name + "`.");
}
+ return Option(reference_field_name);
+}
+
+Option<bool> Collection::get_reference_filter_ids(const std::string & filter_query,
+ const std::string & collection_name,
+ std::pair<uint32_t, uint32_t*>& reference_index_ids) const {
+ auto reference_field_op = get_reference_field(collection_name);
+ if (!reference_field_op.ok()) {
+ return Option<bool>(reference_field_op.code(), reference_field_op.error());
+ }
+
+ std::shared_lock lock(mutex);
+
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
filter_node_t* filter_tree_root = nullptr;
Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,
@@ -2446,8 +2493,8 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que
}
// Reference helper field has the sequence id of other collection's documents.
- reference_field_name += REFERENCE_HELPER_FIELD_SUFFIX;
- index->do_reference_filtering_with_lock(reference_index_ids, filter_tree_root, reference_field_name);
+ auto field_name = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX;
+ index->do_reference_filtering_with_lock(reference_index_ids, filter_tree_root, field_name);
delete filter_tree_root;
return Option<bool>(true);
@@ -3690,10 +3737,12 @@ void Collection::remove_flat_fields(nlohmann::json& document) {
}
}
-void Collection::prune_doc(nlohmann::json& doc,
+Option<bool> Collection::prune_doc(nlohmann::json& doc,
const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names,
- const std::string& parent_name, size_t depth) {
+ const std::string& parent_name, size_t depth,
+ const uint32_t doc_sequence_id, const std::string& collection_name,
+ const std::map<std::string, std::string>& reference_filter_map) {
// doc can only be an object
auto it = doc.begin();
while(it != doc.end()) {
@@ -3768,6 +3817,79 @@ void Collection::prune_doc(nlohmann::json& doc,
it++;
}
+
+ auto reference_it = include_names.equal_prefix_range("$");
+ for (auto reference = reference_it.first; reference != reference_it.second; reference++) {
+ auto ref = reference.key();
+ size_t parenthesis_index = ref.find('(');
+
+ auto ref_collection_name = ref.substr(1, parenthesis_index - 1);
+ auto reference_fields = ref.substr(parenthesis_index + 1, ref.size() - parenthesis_index - 2);
+
+ auto& cm = CollectionManager::get_instance();
+ auto collection = cm.get_collection(ref_collection_name);
+ if (collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` not found.");
+ }
+
+ std::vector<std::string> include_fields_vec;
+ StringUtils::split(reference_fields, include_fields_vec, ",");
+
+ spp::sparse_hash_set<std::string> include_fields, exclude_fields;
+ include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());
+
+ tsl::htrie_set<char> include_fields_full, exclude_fields_full;
+ auto include_exclude_op = collection->populate_include_exclude_fields(include_fields, exclude_fields,
+ include_fields_full, exclude_fields_full);
+ if (!include_exclude_op.ok()) {
+ return include_exclude_op;
+ }
+
+ auto reference_field_op = collection->get_reference_field(collection_name);
+ if (!reference_field_op.ok()) {
+ return Option<bool>(reference_field_op.code(), reference_field_op.error());
+ }
+
+ std::vector<std::pair<size_t, uint32_t*>> documents;
+ auto filter = reference_field_op.get() + REFERENCE_HELPER_FIELD_SUFFIX + ":=" + std::to_string(doc_sequence_id);
+ if (reference_filter_map.count(ref_collection_name) > 0) {
+ filter += "&&";
+ filter += reference_filter_map.at(ref_collection_name);
+ }
+ auto filter_op = collection->get_filter_ids(filter, documents);
+ if (!filter_op.ok()) {
+ return filter_op;
+ }
+
+ if (documents[0].first == 0) {
+ continue;
+ }
+
+ std::vector<nlohmann::json> reference_docs;
+ reference_docs.reserve(documents[0].first);
+ for (size_t i = 0; i < documents[0].first; i++) {
+ auto doc_seq_id = documents[0].second[i];
+
+ nlohmann::json ref_doc;
+ auto get_doc_op = collection->get_document_from_store(doc_seq_id, ref_doc);
+ if (!get_doc_op.ok()) {
+ return get_doc_op;
+ }
+
+ auto prune_op = prune_doc(ref_doc, include_fields_full, exclude_fields_full);
+ if (!prune_op.ok()) {
+ return prune_op;
+ }
+
+ reference_docs.push_back(ref_doc);
+ }
+
+ for (const auto &ref_doc: reference_docs) {
+ doc.update(ref_doc);
+ }
+ }
+
+ return Option<bool>(true);
}
Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 01853fd3..97a7954b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -913,6 +913,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
if(key == FACET_BY){
StringUtils::split_facet(val, *find_str_list_it->second);
}
+ else if(key == INCLUDE_FIELDS){
+ auto op = StringUtils::split_include_fields(val, *find_str_list_it->second);
+ if (!op.ok()) {
+ return op;
+ }
+ }
else{
StringUtils::split(val, *find_str_list_it->second, ",");
}
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index a9409400..9fa65b4c 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -453,6 +453,42 @@ Option<bool> StringUtils::tokenize_filter_query(const std::string& filter_query,
return Option<bool>(true);
}
+Option<bool> StringUtils::split_include_fields(const std::string& include_fields, std::vector<std::string>& tokens) {
+ size_t start = 0, end = 0, size = include_fields.size();
+ std::string include_field;
+
+ while (true) {
+ auto range_pos = include_fields.find('$', start);
+ auto comma_pos = include_fields.find(',', start);
+
+ if (range_pos == std::string::npos && comma_pos == std::string::npos) {
+ if (start < size - 1) {
+ tokens.push_back(include_fields.substr(start, size - start));
+ }
+ break;
+ } else if (range_pos < comma_pos) {
+ end = include_fields.find(')', range_pos);
+ if (end == std::string::npos || end < include_fields.find('(', range_pos)) {
+ return Option<bool>(400, "Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.");
+ }
+
+ include_field = include_fields.substr(range_pos, (end - range_pos) + 1);
+ } else {
+ end = comma_pos;
+ include_field = include_fields.substr(start, end - start);
+ }
+
+ include_field = trim(include_field);
+ if (!include_field.empty()) {
+ tokens.push_back(include_field);
+ }
+
+ start = end + 1;
+ }
+
+ return Option<bool>(true);
+}
+
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
return utf8conv.from_bytes(bytes).size();
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 98e7663f..e96f7f57 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -549,4 +549,164 @@ TEST_F(CollectionJoinTest, FilterByReference_MultipleMatch) {
collectionManager.drop_collection("Users");
collectionManager.drop_collection("Repos");
collectionManager.drop_collection("Links");
+}
+
+TEST_F(CollectionJoinTest, IncludeFieldsByReference_SingleMatch) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_id", "type": "string"},
+ {"name": "product_name", "type": "string"},
+ {"name": "product_description", "type": "string"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "product_id": "product_a",
+ "product_name": "shampoo",
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."
+ })"_json,
+ R"({
+ "product_id": "product_b",
+ "product_name": "soap",
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."
+ })"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Customers",
+ "fields": [
+ {"name": "customer_id", "type": "string"},
+ {"name": "customer_name", "type": "string"},
+ {"name": "product_price", "type": "float"},
+ {"name": "product_id", "type": "string", "reference": "Products.product_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 143,
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 73.5,
+ "product_id": "product_b"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 75,
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 140,
+ "product_id": "product_b"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ std::map<std::string, std::string> req_params = {
+ {"collection", "Products"},
+ {"q", "s"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
+ };
+
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ req_params["include_fields"] = "$foo.bar";
+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());
+
+ req_params["include_fields"] = "$foo(bar";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Invalid reference in include_fields, expected `$CollectionName(fieldA, ...)`.", search_op.error());
+
+ req_params["include_fields"] = "$foo(bar)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Referenced collection `foo` not found.", search_op.error());
+
+ req_params["include_fields"] = "$Customers(bar)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ nlohmann::json res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(0, res_obj["hits"][0]["document"].size());
+
+ req_params["include_fields"] = "$Customers(product_price)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+
+ req_params["include_fields"] = "$Customers(product_price, customer_id)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));
+ ASSERT_EQ("customer_a", res_obj["hits"][0]["document"].at("customer_id"));
+
+ req_params["include_fields"] = "*, $Customers(product_price, customer_id)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ // 3 fields in Products document and 2 fields from Customers document
+ ASSERT_EQ(5, res_obj["hits"][0]["document"].size());
+
+ req_params["include_fields"] = "*, $Customers(product*)";
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ // 3 fields in Products document and 2 fields from Customers document
+ ASSERT_EQ(5, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));
}
\ No newline at end of file