Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support `as` to specify an alias for reference doc's fields. Fix reference `include_fields` returning flattened fields.
Harpreet Sangaree05ecefa1bb
diff --git a/include/collection.h b/include/collection.hindex 9d2202bc..4f8cd435 100644--- a/include/collection.h+++ b/include/collection.h@@ -386,6 +386,7 @@ public:static Option<bool> add_reference_fields(nlohmann::json& doc,Collection *const ref_collection,+ const std::string& alias,const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,@@ -396,7 +397,7 @@ public:size_t depth = 0,const std::map<std::string, reference_filter_result_t>& reference_filter_results = {},Collection *const collection = nullptr, const uint32_t& seq_id = 0,- const std::vector<std::string>& ref_include_fields_vec = {});+ const std::vector<ref_include_fields>& ref_include_fields_vec = {});const Index* _get_index() const;@@ -496,7 +497,7 @@ public:const size_t remote_embedding_num_tries = 2,const std::string& stopwords_set="",const std::vector<std::string>& facet_return_parent = {},- const std::vector<std::string>& ref_include_fields_vec = {}) const;+ const std::vector<ref_include_fields>& ref_include_fields_vec = {}) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/field.h b/include/field.hindex aa54842f..e8fcd1e7 100644--- a/include/field.h+++ b/include/field.h@@ -487,6 +487,11 @@ namespace sort_field_const {static const std::string vector_distance = "_vector_distance";}+struct ref_include_fields {+ std::string expression;+ std::string alias;+};+struct sort_by {enum missing_values_t {first,diff --git a/src/collection.cpp b/src/collection.cppindex 192fd036..7496d680 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1389,7 +1389,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t remote_embedding_num_tries,const std::string& stopwords_set,const std::vector<std::string>& facet_return_parent,- const std::vector<std::string>& ref_include_fields_vec) const {+ const std::vector<ref_include_fields>& ref_include_fields_vec) const {std::shared_lock lock(mutex);@@ -4314,6 +4314,7 @@ void Collection::remove_flat_fields(nlohmann::json& document) {Option<bool> Collection::add_reference_fields(nlohmann::json& doc,Collection *const ref_collection,+ const std::string& alias,const reference_filter_result_t& references,const tsl::htrie_set<char>& ref_include_fields_full,const tsl::htrie_set<char>& ref_exclude_fields_full,@@ -4328,11 +4329,21 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());}+ remove_flat_fields(ref_doc);+auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);if (!prune_op.ok()) {return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }+ }+doc.update(ref_doc);return Option<bool>(true);}@@ -4347,11 +4358,21 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,return Option<bool>(get_doc_op.code(), error_prefix + get_doc_op.error());}+ remove_flat_fields(ref_doc);+auto prune_op = prune_doc(ref_doc, ref_include_fields_full, ref_exclude_fields_full);if (!prune_op.ok()) {return Option<bool>(prune_op.code(), error_prefix + prune_op.error());}+ if (!alias.empty()) {+ auto temp_doc = ref_doc;+ ref_doc.clear();+ for (const auto &item: temp_doc.items()) {+ ref_doc[alias + item.key()] = item.value();+ }+ }+for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {// Add the values of ref_doc as JSON array into doc.doc[ref_doc_it.key()] += ref_doc_it.value();@@ -4367,7 +4388,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,const std::string& parent_name, size_t depth,const std::map<std::string, reference_filter_result_t>& reference_filter_results,Collection *const collection, const uint32_t& seq_id,- const std::vector<std::string>& ref_includes) {+ const std::vector<ref_include_fields>& ref_includes) {// doc can only be an objectauto it = doc.begin();while(it != doc.end()) {@@ -4443,7 +4464,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,it++;}- for (auto const& ref: ref_includes) {+ for (auto const& ref_include: ref_includes) {+ auto const& ref = ref_include.expression;size_t parenthesis_index = ref.find('(');auto ref_collection_name = ref.substr(1, parenthesis_index - 1);@@ -4507,7 +4529,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,Option<bool> add_reference_fields_op = Option<bool>(true);if (has_filter_reference) {- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(),+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias,reference_filter_results.at(ref_collection_name),ref_include_fields_full, ref_exclude_fields_full,error_prefix);@@ -4523,7 +4545,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,}reference_filter_result_t r{1, new uint32[1]{get_reference_doc_id_op.get()}};- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), r,+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,ref_include_fields_full, ref_exclude_fields_full,error_prefix);} else if (joined_coll_has_reference) {@@ -4552,7 +4574,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,r.docs[i] = op.get();}- add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), r,+ add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, r,ref_include_fields_full, ref_exclude_fields_full,error_prefix);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0c1793ea..f7727688 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -9,6 +9,7 @@#include "logger.h"#include "magic_enum.hpp"#include "stopwords_manager.h"+#include "field.h"constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;@@ -838,43 +839,50 @@ void CollectionManager::_get_reference_collection_names(const std::string& filte// Separate out the reference includes into `ref_include_fields_vec`.void initialize_ref_include_fields_vec(const std::string& filter_query, std::vector<std::string>& include_fields_vec,- std::vector<std::string>& ref_include_fields_vec) {+ std::vector<ref_include_fields>& ref_include_fields_vec) {std::set<std::string> reference_collection_names;CollectionManager::_get_reference_collection_names(filter_query, reference_collection_names);std::vector<std::string> result_include_fields_vec;auto wildcard_include_all = true;- for (auto include_field: include_fields_vec) {- if (include_field[0] != '$') {- if (include_field == "*") {+ for (auto include_field_exp: include_fields_vec) {+ if (include_field_exp[0] != '$') {+ if (include_field_exp == "*") {continue;}wildcard_include_all = false;- result_include_fields_vec.emplace_back(include_field);+ result_include_fields_vec.emplace_back(include_field_exp);continue;}- auto open_paren_pos = include_field.find('(');+ auto as_pos = include_field_exp.find(" as ");+ auto ref_include = include_field_exp.substr(0, as_pos),+ alias = (as_pos == std::string::npos) ? "" :+ include_field_exp.substr(as_pos + 4, include_field_exp.size() - (as_pos + 4));++ // For an alias `foo`, we need append `foo.` to all the top level keys of reference doc.+ ref_include_fields_vec.emplace_back(ref_include_fields{ref_include, alias.empty() ? alias :+ StringUtils::trim(alias) + "."});++ auto open_paren_pos = include_field_exp.find('(');if (open_paren_pos == std::string::npos) {continue;}- auto reference_collection_name = include_field.substr(1, open_paren_pos - 1);+ auto reference_collection_name = include_field_exp.substr(1, open_paren_pos - 1);StringUtils::trim(reference_collection_name);if (reference_collection_name.empty()) {continue;}- ref_include_fields_vec.emplace_back(include_field);-// Referenced collection in filter_query is already mentioned in ref_include_fields.reference_collection_names.erase(reference_collection_name);}// Get all the fields of the referenced collection in the filter but not mentioned in include_fields.for (const auto &reference_collection_name: reference_collection_names) {- ref_include_fields_vec.emplace_back("$" + reference_collection_name + "(*)");+ ref_include_fields_vec.emplace_back(ref_include_fields{"$" + reference_collection_name + "(*)", ""});}// Since no field of the collection is mentioned in include_fields, get all the fields.@@ -1048,7 +1056,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::vector<std::string> include_fields_vec;std::vector<std::string> exclude_fields_vec;- std::vector<std::string> ref_include_fields_vec;+ std::vector<ref_include_fields> ref_include_fields_vec;spp::sparse_hash_set<std::string> include_fields;spp::sparse_hash_set<std::string> exclude_fields;diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 6b320fc2..a9296cd5 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -516,6 +516,14 @@ Option<bool> StringUtils::split_include_fields(const std::string& include_fields}include_field = include_fields.substr(range_pos, (end - range_pos) + 1);++ comma_pos = include_fields.find(',', end);+ auto as_pos = include_fields.find(" as ", end);+ if (as_pos != std::string::npos && as_pos < comma_pos) {+ auto alias = include_fields.substr(as_pos, (comma_pos - as_pos));+ end += alias.size() + 1;+ include_field += (" " + trim(alias));+ }} else {end = comma_pos;include_field = include_fields.substr(start, end - start);diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex e1a6e2e4..bc8dea8c 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -1511,6 +1511,26 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ // Add alias using `as`+ req_params = {+ {"collection", "Customers"},+ {"q", "Joe"},+ {"query_by", "customer_name"},+ {"filter_by", "product_price:<100"},+ {"include_fields", "$Products(product_name) as p, product_price"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("p.product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("p.product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+schema_json =R"({"name": "Users",@@ -1649,13 +1669,19 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {"name": "Organizations","fields": [{"name": "org_id", "type": "string"},- {"name": "org_name", "type": "string"}- ]+ {"name": "name", "type": "object"},+ {"name": "name.first", "type": "string"},+ {"name": "name.last", "type": "string"}+ ],+ "enable_nested_fields": true})"_json;documents = {R"({"org_id": "org_a",- "org_name": "Typesense"+ "name": {+ "first": "type",+ "last": "sense"+ }})"_json};collection_create_op = collectionManager.create_collection(schema_json);@@ -1708,7 +1734,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"q", "R"},{"query_by", "user_name"},{"filter_by", "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)"},- {"include_fields", "user_id, user_name, $Repos(repo_content), $Organizations(org_name)"},+ {"include_fields", "user_id, user_name, $Repos(repo_content), $Organizations(name) as org"},{"exclude_fields", "$Participants(*), $Links(*), "}};search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);@@ -1718,14 +1744,18 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(2, res_obj["found"].get<size_t>());ASSERT_EQ(2, res_obj["hits"].size());ASSERT_EQ(4, res_obj["hits"][0]["document"].size());+ASSERT_EQ("user_b", res_obj["hits"][0]["document"].at("user_id"));ASSERT_EQ("Ruby", res_obj["hits"][0]["document"].at("user_name"));ASSERT_EQ("body2", res_obj["hits"][0]["document"].at("repo_content"));- ASSERT_EQ("Typesense", res_obj["hits"][0]["document"].at("org_name"));+ ASSERT_EQ("type", res_obj["hits"][0]["document"]["org.name"].at("first"));+ ASSERT_EQ("sense", res_obj["hits"][0]["document"]["org.name"].at("last"));+ASSERT_EQ("user_a", res_obj["hits"][1]["document"].at("user_id"));ASSERT_EQ("Roshan", res_obj["hits"][1]["document"].at("user_name"));ASSERT_EQ("body2", res_obj["hits"][1]["document"].at("repo_content"));- ASSERT_EQ("Typesense", res_obj["hits"][1]["document"].at("org_name"));+ ASSERT_EQ("type", res_obj["hits"][0]["document"]["org.name"].at("first"));+ ASSERT_EQ("sense", res_obj["hits"][0]["document"]["org.name"].at("last"));}TEST_F(CollectionJoinTest, CascadeDeletion) {diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 1ee9e6ac..bd33a169 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -419,4 +419,12 @@ TEST(StringUtilsTest, SplitIncludeFields) {include_fields = "id, $Collection(title, pref*), count, ";tokens = {"id", "$Collection(title, pref*)", "count"};splitIncludeTestHelper(include_fields, tokens);++ include_fields = "$Collection(title, pref*) as coll";+ tokens = {"$Collection(title, pref*) as coll"};+ splitIncludeTestHelper(include_fields, tokens);++ include_fields = "id, $Collection(title, pref*) as coll , count, ";+ tokens = {"id", "$Collection(title, pref*) as coll", "count"};+ splitIncludeTestHelper(include_fields, tokens);}