Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle per-search API key for multi search.
Kishore Nallan31c8a030c743
diff --git a/include/auth_manager.h b/include/auth_manager.hindex 1aa28fca..94c8704a 100644--- a/include/auth_manager.h+++ b/include/auth_manager.h@@ -5,6 +5,7 @@#include <map>#include <mutex>#include <shared_mutex>+#include <tsl/htrie_map.h>#include "json.hpp"#include "option.h"#include "store.h"@@ -76,13 +77,23 @@ struct api_key_t {}};+struct collection_key_t {+ std::string collection;+ std::string api_key;++ explicit collection_key_t(const std::string& collection, const std::string& api_key):+ collection(collection), api_key(api_key) {++ }+};+class AuthManager {private:mutable std::shared_mutex mutex;- std::map<std::string, api_key_t> api_keys; // stores key_value => key mapping+ tsl::htrie_map<char, api_key_t> api_keys; // stores key_value => key mappingStore *store;std::string bootstrap_auth_key;@@ -100,11 +111,10 @@ private:static std::string fmt_error(std::string&& error, const std::string& key);- Option<bool> authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,- const std::vector<std::string>& collections,+ Option<bool> authenticate_parse_params(const collection_key_t& scoped_api_key, const std::string& action,nlohmann::json& embedded_params) const ;- bool auth_against_key(const std::vector<std::string>& collections, const std::string& action,+ bool auth_against_key(const std::string& req_collection, const std::string& action,const api_key_t &api_key, const bool search_only) const;public:@@ -124,8 +134,10 @@ public:Option<api_key_t> remove_key(uint32_t id);- bool authenticate(const std::string& req_api_key, const std::string& action,- const std::vector<std::string>& collections, std::map<std::string, std::string>& params) const;+ bool authenticate(const std::string& action,+ const std::vector<collection_key_t>& collection_keys,+ std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec) const;static bool add_item_to_params(std::map<std::string, std::string> &req_params,const nlohmann::detail::iteration_proxy_value<nlohmann::json::iterator>& item,diff --git a/include/collection_manager.h b/include/collection_manager.hindex f38b1c3f..c0e1dbd0 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -137,8 +137,10 @@ public:// frees in-memory data structures when server is shutdown - helps us run a memory leak detector properlyvoid dispose();- bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,- const std::vector<std::string>& collections, std::map<std::string, std::string>& params) const;+ bool auth_key_matches(const string& req_auth_key, const string& action,+ const std::vector<collection_key_t>& collection_keys,+ std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec) const;static Option<Collection*> create_collection(nlohmann::json& req_json);@@ -170,7 +172,9 @@ public:AuthManager& getAuthManager();- static Option<bool> do_search(std::map<std::string, std::string>& req_params, std::string& results_json_str);+ static Option<bool> do_search(std::map<std::string, std::string>& req_params,+ nlohmann::json& embedded_params,+ std::string& results_json_str);static bool parse_sort_by_str(std::string sort_by_str, std::vector<sort_by>& sort_fields);diff --git a/include/core_api.h b/include/core_api.hindex 91862be7..3419b802 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -1,9 +1,12 @@#pragma once#include "http_server.h"+#include "auth_manager.h"-bool handle_authentication(std::map<std::string, std::string>& req_params, const std::string& body, const route_path& rpath,- const std::string& auth_key);+bool handle_authentication(std::map<std::string, std::string>& req_params,+ std::vector<nlohmann::json>& embedded_params_vec,+ const std::string& body, const route_path& rpath,+ const std::string& req_auth_key);// Collections@@ -112,7 +115,8 @@ bool post_clear_cache(const std::shared_ptr<http_req>& req, const std::shared_pt// Misc helpersvoid get_collections_for_auth(std::map<std::string, std::string> &req_params, const std::string &body,- const route_path &rpath, std::vector<std::string> &collections);+ const route_path &rpath, const std::string& req_auth_key,+ std::vector<collection_key_t> &collections);bool is_doc_import_route(uint64_t route_hash);diff --git a/include/http_data.h b/include/http_data.hindex 8588dfe5..97cc5a33 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -203,6 +203,7 @@ struct http_req {std::string path_without_query;uint64_t route_hash;std::map<std::string, std::string> params;+ std::vector<nlohmann::json> embedded_params_vec;bool first_chunk_aggregate;std::atomic<bool> last_chunk_aggregate;@@ -237,9 +238,11 @@ struct http_req {}http_req(h2o_req_t* _req, const std::string & http_method, const std::string & path_without_query, uint64_t route_hash,- const std::map<std::string, std::string> & params, const std::string& body):+ const std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec, const std::string& body):_req(_req), http_method(http_method), path_without_query(path_without_query), route_hash(route_hash),- params(params), first_chunk_aggregate(true), last_chunk_aggregate(false),+ params(params), embedded_params_vec(embedded_params_vec),+ first_chunk_aggregate(true), last_chunk_aggregate(false),chunk_len(0), body(body), body_index(0), data(nullptr), ready(false),log_index(0) {diff --git a/include/http_server.h b/include/http_server.hindex e1a60be2..0298718e 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -198,7 +198,9 @@ private:ThreadPool* thread_pool;- bool (*auth_handler)(std::map<std::string, std::string>& params, const std::string& body, const route_path& rpath,+ bool (*auth_handler)(std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec,+ const std::string& body, const route_path& rpath,const std::string& auth_key);static void on_accept(h2o_socket_t *listener, const char *err);@@ -253,7 +255,8 @@ public:uint64_t node_state() const;- void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const std::string& body,+ void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec, const std::string& body,const route_path & rpath, const std::string & auth_key));void get(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res), bool async_req=false, bool async_res=false);diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex cb24b6f4..2f812ed7 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -139,41 +139,41 @@ uint32_t AuthManager::get_next_api_key_id() {return next_api_key_id++;}-bool AuthManager::authenticate(const std::string& req_api_key, const std::string& action,- const std::vector<std::string>& collections,- std::map<std::string, std::string>& params) const {+bool AuthManager::authenticate(const std::string& action,+ const std::vector<collection_key_t>& collection_keys,+ std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec) const {std::shared_lock lock(mutex);//LOG(INFO) << "AuthManager::authenticate()";- const auto& key_it = api_keys.find(req_api_key);- if(key_it != api_keys.end()) {- const api_key_t& api_key = key_it->second;- return auth_against_key(collections, action, api_key, false);- }-- // could be a scoped API key- nlohmann::json embedded_params;- Option<bool> auth_op = authenticate_parse_params(req_api_key, action, collections, embedded_params);- if(!auth_op.ok()) {- return false;- }+ size_t num_keys_matched = 0;+ for(const auto& coll_key: collection_keys) {+ const auto& key_it = api_keys.find(coll_key.api_key);+ nlohmann::json embedded_params;- // enrich params with values from embedded_params- for(auto& item: embedded_params.items()) {- if(item.key() == "expires_at") {- continue;+ if(key_it != api_keys.end()) {+ const api_key_t& api_key = key_it.value();+ if(!auth_against_key(coll_key.collection, action, api_key, false)) {+ return false;+ }+ } else {+ // could be a scoped API key+ Option<bool> auth_op = authenticate_parse_params(coll_key, action, embedded_params);+ if(!auth_op.ok()) {+ return false;+ }}- // overwrite = true as embedded params have higher priority- AuthManager::add_item_to_params(params, item, true);+ num_keys_matched++;+ embedded_params_vec.push_back(embedded_params);}//LOG(INFO) << "api_keys.size() = " << api_keys.size();- return true;+ return (num_keys_matched == collection_keys.size());}-bool AuthManager::auth_against_key(const std::vector<std::string>& collections, const std::string& action,+bool AuthManager::auth_against_key(const std::string& req_collection, const std::string& action,const api_key_t& api_key, const bool search_only) const {if(uint64_t(std::time(0)) > api_key.expires_at) {@@ -214,28 +214,25 @@ bool AuthManager::auth_against_key(const std::vector<std::string>& collections,}}- for(const std::string& req_collection: collections) {- bool coll_allowed = false;+ bool coll_allowed = false;- for(const std::string& allowed_collection: api_key.collections) {- if(allowed_collection == "*" || (allowed_collection == req_collection) || req_collection.empty() ||- std::regex_match (req_collection, std::regex(allowed_collection))) {- coll_allowed = true;- break;- }+ for(const std::string& allowed_collection: api_key.collections) {+ if(allowed_collection == "*" || (allowed_collection == req_collection) || req_collection.empty() ||+ std::regex_match (req_collection, std::regex(allowed_collection))) {+ coll_allowed = true;+ break;}+ }- if(!coll_allowed) {- // even if one collection is not allowed, we reject the entire request- return false;- }+ if(!coll_allowed) {+ // even if one collection is not allowed, we reject the entire request+ return false;}return true;}-Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,- const std::vector<std::string>& collections,+Option<bool> AuthManager::authenticate_parse_params(const collection_key_t& scoped_api_key, const std::string& action,nlohmann::json& embedded_params) const {// allow only searches from scoped keys@@ -244,7 +241,7 @@ Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_apreturn Option<bool>(403, "Forbidden.");}- const std::string& key_payload = StringUtils::base64_decode(scoped_api_key);+ const std::string& key_payload = StringUtils::base64_decode(scoped_api_key.api_key);if(key_payload.size() < HMAC_BASE64_LEN + api_key_t::PREFIX_LEN) {LOG(ERROR) << "Malformed scoped API key.";@@ -258,51 +255,51 @@ Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_apconst std::string& custom_params = key_payload.substr(HMAC_BASE64_LEN + api_key_t::PREFIX_LEN);// calculate and verify hmac against matching api key- for (const auto &kv : api_keys) {- if(kv.first.substr(0, api_key_t::PREFIX_LEN) == key_prefix) {- const api_key_t& api_key = kv.second;+ auto prefix_range = api_keys.equal_prefix_range(key_prefix);- // ensure that parent key collection filter matches queried collection- bool auth_success = auth_against_key(collections, action, api_key, true);+ for(auto it = prefix_range.first; it != prefix_range.second; ++it) {+ const api_key_t& root_api_key = it.value();- if(!auth_success) {- LOG(ERROR) << fmt_error("Parent key does not allow queries against queried collection.", api_key.value);+ // ensure that parent key collection filter matches queried collection+ bool auth_success = auth_against_key(scoped_api_key.collection, action, root_api_key, true);++ if(!auth_success) {+ LOG(ERROR) << fmt_error("Parent key does not allow queries against queried collection.", root_api_key.value);+ return Option<bool>(403, "Forbidden.");+ }++ // finally verify hmac+ std::string digest = StringUtils::hmac(root_api_key.value, custom_params);++ if(digest == hmacSHA256) {+ try {+ embedded_params = nlohmann::json::parse(custom_params);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();return Option<bool>(403, "Forbidden.");}- // finally verify hmac- std::string digest = StringUtils::hmac(kv.first, custom_params);-- if(digest == hmacSHA256) {- try {- embedded_params = nlohmann::json::parse(custom_params);- } catch(const std::exception& e) {- LOG(ERROR) << "JSON error: " << e.what();- return Option<bool>(403, "Forbidden.");- }+ if(!embedded_params.is_object()) {+ LOG(ERROR) << fmt_error("Scoped API key contains invalid search parameters.", root_api_key.value);+ return Option<bool>(403, "Forbidden.");+ }- if(!embedded_params.is_object()) {- LOG(ERROR) << fmt_error("Scoped API key contains invalid search parameters.", api_key.value);+ if(embedded_params.count("expires_at") != 0) {+ if(!embedded_params["expires_at"].is_number_integer() || embedded_params["expires_at"].get<int64_t>() < 0) {+ LOG(ERROR) << fmt_error("Wrong format for `expires_at`. It should be an unsigned integer.", root_api_key.value);return Option<bool>(403, "Forbidden.");}- if(embedded_params.count("expires_at") != 0) {- if(!embedded_params["expires_at"].is_number_integer() || embedded_params["expires_at"].get<int64_t>() < 0) {- LOG(ERROR) << fmt_error("Wrong format for `expires_at`. It should be an unsigned integer.", api_key.value);- return Option<bool>(403, "Forbidden.");- }-- // if parent key's expiry timestamp is smaller, it takes precedence- uint64_t expiry_ts = std::min(api_key.expires_at, embedded_params["expires_at"].get<uint64_t>());+ // if parent key's expiry timestamp is smaller, it takes precedence+ uint64_t expiry_ts = std::min(root_api_key.expires_at, embedded_params["expires_at"].get<uint64_t>());- if(uint64_t(std::time(0)) > expiry_ts) {- LOG(ERROR) << fmt_error("Scoped API key has expired. ", api_key.value);- return Option<bool>(403, "Forbidden.");- }+ if(uint64_t(std::time(0)) > expiry_ts) {+ LOG(ERROR) << fmt_error("Scoped API key has expired. ", root_api_key.value);+ return Option<bool>(403, "Forbidden.");}-- return Option<bool>(true);}++ return Option<bool>(true);}}diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 5f66c8b6..f1857814 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -106,7 +106,7 @@ std::string BatchedIndexer::get_collection_name(const std::shared_ptr<http_req>&if(rpath->handler == post_create_collection) {nlohmann::json obj = nlohmann::json::parse(req->body, nullptr, false);- if(obj != nlohmann::json::value_t::discarded && obj.is_object() &&+ if(!obj.is_discarded() && obj.is_object() &&obj.count("name") != 0 && obj["name"].is_string()) {coll_name = obj["name"];}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 78439bc7..bb1c6729 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -168,7 +168,7 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sconst auto& collection_meta_json = collection_meta_jsons[coll_index];nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json, nullptr, false);- if(collection_meta == nlohmann::json::value_t::discarded) {+ if(collection_meta.is_discarded()) {LOG(ERROR) << "Error while parsing collection meta, json: " << collection_meta_json;return Option<bool>(500, "Error while parsing collection meta.");}@@ -265,23 +265,23 @@ void CollectionManager::dispose() {store->close();}-bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,- const std::string& action,- const std::vector<std::string>& collections,- std::map<std::string, std::string>& params) const {+bool CollectionManager::auth_key_matches(const string& req_auth_key, const string& action,+ const std::vector<collection_key_t>& collection_keys,+ std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec) const {std::shared_lock lock(mutex);- if(auth_key_sent.empty()) {+ if(req_auth_key.empty()) {return false;}// check with bootstrap auth key- if(bootstrap_auth_key == auth_key_sent) {+ if(bootstrap_auth_key == req_auth_key) {return true;}// finally, check managed auth keys- return auth_manager.authenticate(auth_key_sent, action, collections, params);+ return auth_manager.authenticate(action, collection_keys, params, embedded_params_vec);}Option<Collection*> CollectionManager::create_collection(const std::string& name,@@ -554,7 +554,9 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<s}-Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& req_params, std::string& results_json_str) {+Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& req_params,+ nlohmann::json& embedded_params,+ std::string& results_json_str) {auto begin = std::chrono::high_resolution_clock::now();const char *NUM_TYPOS = "num_typos";@@ -615,6 +617,16 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *PRESET = "preset";+ // enrich params with values from embedded params+ for(auto& item: embedded_params.items()) {+ if(item.key() == "expires_at") {+ continue;+ }++ // overwrite = true as embedded params have higher priority+ AuthManager::add_item_to_params(req_params, item, true);+ }+if(req_params.count(NUM_TYPOS) == 0) {req_params[NUM_TYPOS] = "2";}diff --git a/src/core_api.cpp b/src/core_api.cppindex 9b976d3d..c949fa95 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -16,20 +16,23 @@ using namespace std::chrono_literals;std::shared_mutex mutex;LRU::Cache<uint64_t, cached_res_t> res_cache;-bool handle_authentication(std::map<std::string, std::string>& req_params, const std::string& body,- const route_path& rpath, const std::string& auth_key) {+bool handle_authentication(std::map<std::string, std::string>& req_params,+ std::vector<nlohmann::json>& embedded_params_vec,+ const std::string& body,+ const route_path& rpath,+ const std::string& req_auth_key) {CollectionManager & collectionManager = CollectionManager::get_instance();- std::vector<std::string> collections;+ std::vector<collection_key_t> collections;- get_collections_for_auth(req_params, body, rpath, collections);+ get_collections_for_auth(req_params, body, rpath, req_auth_key, collections);if(rpath.handler == get_health) {// health endpoint requires no authenticationreturn true;}- return collectionManager.auth_key_matches(auth_key, rpath.action, collections, req_params);+ return collectionManager.auth_key_matches(req_auth_key, rpath.action, collections, req_params, embedded_params_vec);}void stream_response(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {@@ -53,40 +56,53 @@ void defer_processing(const std::shared_ptr<http_req>& req, const std::shared_ptserver->get_message_dispatcher()->send_message(HttpServer::DEFER_PROCESSING_MESSAGE, defer);}-void get_collections_for_auth(std::map<std::string, std::string> &req_params, const std::string &body,- const route_path &rpath, std::vector<std::string> &collections) {- if(req_params.count("collection") != 0) {- collections.emplace_back(req_params.at("collection"));- }+void get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,+ const route_path& rpath, const std::string& req_auth_key,+ std::vector<collection_key_t>& collections) {if(rpath.handler == post_multi_search) {nlohmann::json obj = nlohmann::json::parse(body, nullptr, false);- if(obj == nlohmann::json::value_t::discarded) {+ if(obj.is_discarded()) {LOG(ERROR) << "Multi search request body is malformed.";+ collections.emplace_back("", req_auth_key);}- if(obj != nlohmann::json::value_t::discarded && obj.count("searches") != 0 && obj["searches"].is_array()) {+ else if(obj.count("searches") != 0 && obj["searches"].is_array()) {for(auto& el : obj["searches"]) {- if(el.is_object() && el.count("collection") != 0) {- collections.emplace_back(el["collection"].get<std::string>());+ if(el.is_object()) {+ std::string coll_name;+ if(el.count("collection") != 0) {+ coll_name = el["collection"].get<std::string>();+ } else if(req_params.count("collection") != 0) {+ coll_name = req_params["collection"];+ }++ const std::string& access_key = (el.count("x-typesense-api-key") != 0) ?+ el["x-typesense-api-key"].get<std::string>() :+ req_auth_key;++ collections.emplace_back(coll_name, access_key);}}}- } else if(rpath.handler == post_create_collection) {- nlohmann::json obj = nlohmann::json::parse(body, nullptr, false);-- if(obj == nlohmann::json::value_t::discarded) {- LOG(ERROR) << "Create collection request body is malformed.";- }-- if(obj != nlohmann::json::value_t::discarded && obj.count("name") != 0 && obj["name"].is_string()) {- collections.emplace_back(obj["name"].get<std::string>());+ } else {+ if(rpath.handler == post_create_collection) {+ nlohmann::json obj = nlohmann::json::parse(body, nullptr, false);++ if(obj.is_discarded()) {+ LOG(ERROR) << "Create collection request body is malformed.";+ collections.emplace_back("", req_auth_key);+ } else if(obj.count("name") != 0 && obj["name"].is_string()) {+ collections.emplace_back(obj["name"].get<std::string>(), req_auth_key);+ }+ } else if(req_params.count("collection") != 0) {+ collections.emplace_back(req_params.at("collection"), req_auth_key);}}- else if(collections.empty()) {- collections.emplace_back("");+ if(collections.empty()) {+ collections.emplace_back("", req_auth_key);}}@@ -286,8 +302,13 @@ bool get_search(const std::shared_ptr<http_req>& req, const std::shared_ptr<http}}+ if(req->embedded_params_vec.empty()) {+ res->set_500("Embedded params is empty.");+ return false;+ }+std::string results_json_str;- Option<bool> search_op = CollectionManager::do_search(req->params, results_json_str);+ Option<bool> search_op = CollectionManager::do_search(req->params, req->embedded_params_vec[0], results_json_str);if(!search_op.ok()) {res->set(search_op.code(), search_op.error());@@ -391,7 +412,9 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_pnlohmann::json& searches = req_json["searches"];- for(auto& search_params: searches) {+ for(size_t i = 0; i < searches.size(); i++) {+ auto& search_params = searches[i];+if(!search_params.is_object()) {res->set_400("The value of `searches` must be an array of objects.");return false;@@ -414,7 +437,7 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p}std::string results_json_str;- Option<bool> search_op = CollectionManager::do_search(req->params, results_json_str);+ Option<bool> search_op = CollectionManager::do_search(req->params, req->embedded_params_vec[i], results_json_str);if(search_op.ok()) {response["results"].push_back(nlohmann::json::parse(results_json_str));diff --git a/src/http_server.cpp b/src/http_server.cppindex 388fd3d6..42a0662f 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -441,8 +441,10 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {}const std::string & body = std::string(req->entity.base, req->entity.len);+ std::vector<nlohmann::json> embedded_params_vec;- bool authenticated = h2o_handler->http_server->auth_handler(query_map, body, *rpath, api_auth_key_sent);+ bool authenticated = h2o_handler->http_server->auth_handler(query_map, embedded_params_vec, body, *rpath,+ api_auth_key_sent);if(!authenticated) {std::string message = std::string("{\"message\": \"Forbidden - a valid `") + http_req::AUTH_HEADER +"` header must be sent.\"}";@@ -450,7 +452,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {}std::shared_ptr<http_req> request = std::make_shared<http_req>(req, rpath->http_method, path_without_query,- route_hash, query_map, body);+ route_hash, query_map, embedded_params_vec, body);// add custom generator with a dispose function for cleaning up resourcesh2o_custom_generator_t* custom_gen = new h2o_custom_generator_t;@@ -806,7 +808,9 @@ void HttpServer::stream_response(stream_response_state_t& state) {//LOG(INFO) << "stream_response after send";}-void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const std::string& body,+void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params,+ std::vector<nlohmann::json>& embedded_params_vec,+ const std::string& body,const route_path& rpath, const std::string& auth_key)) {auth_handler = handler;}diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cppindex 16aa2018..a5c97044 100644--- a/test/auth_manager_test.cpp+++ b/test/auth_manager_test.cpp@@ -138,98 +138,119 @@ TEST_F(AuthManagerTest, CheckRestoreOfAPIKeys) {TEST_F(AuthManagerTest, VerifyAuthentication) {std::map<std::string, std::string> sparams;+ std::vector<nlohmann::json> embedded_params;// when no keys are present at all- ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "", {}, sparams));+ ASSERT_FALSE(auth_manager.authenticate("", {collection_key_t("", "jdlaslasdasd")}, sparams, embedded_params));// wildcard permissionapi_key_t wildcard_all_key = api_key_t("abcd1", "wildcard all key", {"*"}, {"*"}, FUTURE_TS);auth_manager.create_key(wildcard_all_key);- ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "documents:create", {"collection1"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(wildcard_all_key.value, "metrics:get", {""}, sparams));+ ASSERT_FALSE(auth_manager.authenticate("documents:create", {collection_key_t("collection1", "jdlaslasdasd")}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("metrics:get", {collection_key_t("", wildcard_all_key.value)}, sparams, embedded_params));// long API keystd::string long_api_key_str = StringUtils::randstring(50);api_key_t long_api_key = api_key_t(long_api_key_str, "long api key", {"*"}, {"*"}, FUTURE_TS);auth_manager.create_key(long_api_key);- ASSERT_TRUE(auth_manager.authenticate(long_api_key_str, "metrics:get", {""}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("metrics:get", {collection_key_t(long_api_key_str, wildcard_all_key.value)}, sparams, embedded_params));// wildcard on a collectionapi_key_t wildcard_coll_key = api_key_t("abcd2", "wildcard coll key", {"*"}, {"collection1"}, FUTURE_TS);auth_manager.create_key(wildcard_coll_key);- ASSERT_FALSE(auth_manager.authenticate("adasda", "documents:create", {"collection1"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", {"collection1"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", {"collection2"}, sparams));+ ASSERT_FALSE(auth_manager.authenticate("documents:create", {collection_key_t("collection1", "adasda")}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("documents:get", {collection_key_t("collection1", wildcard_coll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:get", {collection_key_t("collection2", wildcard_coll_key.value)}, sparams, embedded_params));// wildcard on multiple collectionsapi_key_t wildcard_colls_key = api_key_t("abcd3", "wildcard coll key", {"*"}, {"collection1", "collection2", "collection3"}, FUTURE_TS);auth_manager.create_key(wildcard_colls_key);- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", {"collection1"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:search", {"collection2"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:create", {"collection3"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", {"collection4"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", {"*"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("documents:get", {collection_key_t("collection1", wildcard_colls_key.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("documents:search", {collection_key_t("collection2", wildcard_colls_key.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("documents:create", {collection_key_t("collection3", wildcard_colls_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:get", {collection_key_t("collection4", wildcard_colls_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:get", {collection_key_t("*", wildcard_colls_key.value)}, sparams, embedded_params));// only 1 action on multiple collectionsapi_key_t one_action_key = api_key_t("abcd4", "one action key", {"documents:search"}, {"collection1", "collection2"}, FUTURE_TS);auth_manager.create_key(one_action_key);- ASSERT_TRUE(auth_manager.authenticate(one_action_key.value, "documents:search", {"collection1"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:get", {"collection2"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:search", {{"collection5"}}, sparams));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "*", {"collection2"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("documents:search", {collection_key_t("collection1", one_action_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:get", {collection_key_t("collection2", one_action_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("collection5", one_action_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("*", {collection_key_t("collection2", one_action_key.value)}, sparams, embedded_params));// multiple actions on multiple collectionsapi_key_t mul_acoll_key = api_key_t("abcd5", "multiple action/collection key",{"documents:get", "collections:list"}, {"metacollection", "collection2"}, FUTURE_TS);auth_manager.create_key(mul_acoll_key);- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", {"metacollection"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", {"collection2"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", {"metacollection"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:search", {"collection2"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", {"collection5"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "*", {"*"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("documents:get", {collection_key_t("metacollection", mul_acoll_key.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("collection2", mul_acoll_key.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("metacollection", mul_acoll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("collection2", mul_acoll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:get", {collection_key_t("collection5", mul_acoll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("*", {collection_key_t("*", mul_acoll_key.value)}, sparams, embedded_params));// regexp matchapi_key_t regexp_colls_key1 = api_key_t("abcd6", "regexp coll key", {"*"}, {"coll.*"}, FUTURE_TS);auth_manager.create_key(regexp_colls_key1);- ASSERT_TRUE(auth_manager.authenticate(regexp_colls_key1.value, "collections:list", {{"collection2"}}, sparams));- ASSERT_TRUE(auth_manager.authenticate(regexp_colls_key1.value, "documents:get", {"collection5"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("collection2", regexp_colls_key1.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("documents:get", {collection_key_t("collection5", regexp_colls_key1.value)}, sparams, embedded_params));api_key_t regexp_colls_key2 = api_key_t("abcd7", "regexp coll key", {"*"}, {".*meta.*"}, FUTURE_TS);auth_manager.create_key(regexp_colls_key2);- ASSERT_TRUE(auth_manager.authenticate(regexp_colls_key2.value, "collections:list", {"metacollection"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(regexp_colls_key2.value, "collections:list", {"ametacollection"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("metacollection", regexp_colls_key2.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("ametacollection", regexp_colls_key2.value)}, sparams, embedded_params));// check for expiryapi_key_t expired_key1 = api_key_t("abcd8", "expiry key", {"*"}, {"*"}, 1606542716);auth_manager.create_key(expired_key1);- ASSERT_FALSE(auth_manager.authenticate(expired_key1.value, "collections:list", {"collection"}, sparams));+ ASSERT_FALSE(auth_manager.authenticate("collections:list", {collection_key_t("collection", expired_key1.value)}, sparams, embedded_params));api_key_t unexpired_key1 = api_key_t("abcd9", "expiry key", {"*"}, {"*"}, 2237712220);auth_manager.create_key(unexpired_key1);- ASSERT_TRUE(auth_manager.authenticate(unexpired_key1.value, "collections:list", {"collection"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("collections:list", {collection_key_t("collection", unexpired_key1.value)}, sparams, embedded_params));// wildcard action on any collectionapi_key_t wildcard_action_coll_key = api_key_t("abcd10", "wildcard coll action key", {"collections:*"}, {"*"}, FUTURE_TS);auth_manager.create_key(wildcard_action_coll_key);- ASSERT_TRUE(auth_manager.authenticate(wildcard_action_coll_key.value, "collections:create", {"collection1"}, sparams));- ASSERT_TRUE(auth_manager.authenticate(wildcard_action_coll_key.value, "collections:delete", {"collection1", "collection2"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(wildcard_action_coll_key.value, "documents:create", {"collection1"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("collections:create", {collection_key_t("collection1", wildcard_action_coll_key.value)}, sparams, embedded_params));+ ASSERT_TRUE(auth_manager.authenticate("collections:delete", {collection_key_t("collection1", wildcard_action_coll_key.value), collection_key_t("collection2", wildcard_action_coll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:create", {collection_key_t("collection1", wildcard_action_coll_key.value)}, sparams, embedded_params));// create action on a specific collectionapi_key_t create_action_coll_key = api_key_t("abcd11", "create action+coll key", {"collections:create"}, {"collection1"}, FUTURE_TS);auth_manager.create_key(create_action_coll_key);- ASSERT_TRUE(auth_manager.authenticate(create_action_coll_key.value, "collections:create", {"collection1"}, sparams));- ASSERT_FALSE(auth_manager.authenticate(create_action_coll_key.value, "collections:create", {"collection2"}, sparams));+ ASSERT_TRUE(auth_manager.authenticate("collections:create", {collection_key_t("collection1", create_action_coll_key.value)}, sparams, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("collections:create", {collection_key_t("collection2", create_action_coll_key.value)}, sparams, embedded_params));++ // two keys against 2 different collections: both should be valid+ api_key_t coll_a_key = api_key_t("coll_a", "one action key", {"documents:search"}, {"collectionA"}, FUTURE_TS);+ api_key_t coll_b_key = api_key_t("coll_b", "one action key", {"documents:search"}, {"collectionB"}, FUTURE_TS);+ auth_manager.create_key(coll_a_key);+ auth_manager.create_key(coll_b_key);+ ASSERT_TRUE(auth_manager.authenticate("documents:search",+ {collection_key_t("collectionA", coll_a_key.value),+ collection_key_t("collectionB", coll_b_key.value)},+ sparams, embedded_params));++ ASSERT_FALSE(auth_manager.authenticate("documents:search",+ {collection_key_t("collectionA", coll_a_key.value),+ collection_key_t("collection1", coll_b_key.value)},+ sparams, embedded_params));++ ASSERT_FALSE(auth_manager.authenticate("documents:search",+ {collection_key_t("collection1", coll_a_key.value),+ collection_key_t("collectionB", coll_b_key.value)},+ sparams, embedded_params));}TEST_F(AuthManagerTest, GenerationOfAPIAction) {@@ -257,36 +278,50 @@ TEST_F(AuthManagerTest, GenerationOfAPIAction) {TEST_F(AuthManagerTest, ScopedAPIKeys) {std::map<std::string, std::string> params;params["filter_by"] = "country:USA";+ std::vector<nlohmann::json> embedded_params;// create a API key bound to search scope and a given collectionapi_key_t key_search_coll1("KeyVal", "test key", {"documents:search"}, {"coll1"}, FUTURE_TS);auth_manager.create_key(key_search_coll1);std::string scoped_key = StringUtils::base64_encode(- "IvjqWNZ5M5ElcvbMoXj45BxkQrZG4ZKEaNQoRioCx2s=KeyV{\"filter_by\": \"user_id:1080\"}"+ R"(IvjqWNZ5M5ElcvbMoXj45BxkQrZG4ZKEaNQoRioCx2s=KeyV{"filter_by": "user_id:1080"})");- ASSERT_TRUE(auth_manager.authenticate(scoped_key, "documents:search", {"coll1"}, params));- ASSERT_STREQ("country:USA&&user_id:1080", params["filter_by"].c_str());+ ASSERT_TRUE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", scoped_key)}, params, embedded_params));+ ASSERT_EQ("user_id:1080", embedded_params[0]["filter_by"].get<std::string>());// should scope to collection bound by the parent key- ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:search", {"coll2"}, params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll2", scoped_key)}, params, embedded_params));// should scope to search action only- ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:create", {"coll1"}, params));+ ASSERT_FALSE(auth_manager.authenticate("documents:create", {collection_key_t("coll1", scoped_key)}, params, embedded_params));// check with corrupted key- ASSERT_FALSE(auth_manager.authenticate("asdasasd", "documents:search", {"coll1"}, params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", "asdasasd")}, params, embedded_params));++ // with multiple collections, all should be authenticated+ ASSERT_FALSE(auth_manager.authenticate("documents:search",+ {collection_key_t("coll1", scoped_key),+ collection_key_t("coll2", scoped_key)},+ params, embedded_params));++ // send both regular key and scoped key+ ASSERT_TRUE(auth_manager.authenticate("documents:search",+ {collection_key_t("coll1", key_search_coll1.value),+ collection_key_t("coll1", scoped_key)},+ params, embedded_params));// when params is empty, embedded param should be setstd::map<std::string, std::string> empty_params;- ASSERT_TRUE(auth_manager.authenticate(scoped_key, "documents:search", {"coll1"}, empty_params));- ASSERT_STREQ("user_id:1080", empty_params["filter_by"].c_str());+ embedded_params.clear();+ ASSERT_TRUE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", scoped_key)}, empty_params, embedded_params));+ ASSERT_EQ("user_id:1080", embedded_params[0]["filter_by"].get<std::string>());// when more than a single key prefix matches, must pick the correct underlying keyapi_key_t key_search_coll2("KeyVal2", "test key", {"documents:search"}, {"coll2"}, FUTURE_TS);auth_manager.create_key(key_search_coll2);- ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:search", {"coll2"}, empty_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll2", scoped_key)}, empty_params, embedded_params));// should only allow scoped API keys derived from parent key with documents:search actionapi_key_t key_search_admin("AdminKey", "admin key", {"*"}, {"*"}, FUTURE_TS);@@ -294,7 +329,7 @@ TEST_F(AuthManagerTest, ScopedAPIKeys) {std::string scoped_key2 = StringUtils::base64_encode("BXbsk+xLT1gxOjDyip6+PE4MtOzOm/H7kbkN1d/j/s4=Admi{\"filter_by\": \"user_id:1080\"}");- ASSERT_FALSE(auth_manager.authenticate(scoped_key2, "documents:search", {"coll2"}, empty_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll2", scoped_key2)}, empty_params, embedded_params));// expiration of scoped api key@@ -303,13 +338,14 @@ TEST_F(AuthManagerTest, ScopedAPIKeys) {auth_manager.create_key(key_expiry);empty_params.clear();+ embedded_params.clear();std::string scoped_key3 = "K1M2STRDelZYNHpxNGVWUTlBTGpOWUl4dk8wNU8xdnVEZi9aSUcvZE5tcz1FeHBpeyJmaWx0ZXJfYnkiOi""AidXNlcl9pZDoxMDgwIiwgImV4cGlyZXNfYXQiOiAyMjM3NzEyMjIwfQ==";- ASSERT_TRUE(auth_manager.authenticate(scoped_key3, "documents:search", {"coll1"}, empty_params));- ASSERT_STREQ("user_id:1080", empty_params["filter_by"].c_str());- ASSERT_EQ(1, empty_params.size());+ ASSERT_TRUE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", scoped_key3)}, empty_params, embedded_params));+ ASSERT_EQ("user_id:1080", embedded_params[0]["filter_by"].get<std::string>());+ ASSERT_EQ(1, embedded_params.size());// {"filter_by": "user_id:1080", "expires_at": 1606563316} (expired)@@ -321,7 +357,7 @@ TEST_F(AuthManagerTest, ScopedAPIKeys) {std::string scoped_key4 = "SXFKNldZZWRiWkVKVmI2RCt3OTlKNHpBZ24yWlRUbEdJdERtTy9IZ2REZz1FeHBpeyJmaWx0ZXJfYnkiOiAidXN""lcl9pZDoxMDgwIiwgImV4cGlyZXNfYXQiOiAxNjA2NTYzMzE2fQ==";- ASSERT_FALSE(auth_manager.authenticate(scoped_key4, "documents:search", {"coll1"}, empty_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", scoped_key4)}, empty_params, embedded_params));// {"filter_by": "user_id:1080", "expires_at": 64723363200} (greater than parent key expiry)// embedded key's param cannot exceed parent's expiry@@ -334,12 +370,12 @@ TEST_F(AuthManagerTest, ScopedAPIKeys) {std::string scoped_key5 = "V3JMNFJlZHRMVStrZHphNFVGZDh4MWltSmx6Yzk2R3QvS2ZwSE8weGRWQT1FeHBpeyJmaWx0ZXJfYnkiOiAidX""Nlcl9pZDoxMDgwIiwgImV4cGlyZXNfYXQiOiA2NDcyMzM2MzIwMH0=";- ASSERT_FALSE(auth_manager.authenticate(scoped_key5, "documents:search", {"coll1"}, empty_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", scoped_key5)}, empty_params, embedded_params));// bad scoped API key- ASSERT_FALSE(auth_manager.authenticate(" XhsdBdhehdDheruyhvbdhwjhHdhgyeHbfheR", "documents:search", {"coll1"}, empty_params));- ASSERT_FALSE(auth_manager.authenticate("cXYPvkNKRlQrBzVTEgY4a3FrZfZ2MEs4kFJ6all3eldwM GhKZnRId3Y3TT1RZmxZeYJmaWx0ZXJfYnkiOkJ1aWQ6OElVm1lUVm15SG9ZOHM4NUx2VFk4S2drNHJIMiJ9", "documents:search", {"coll1"}, empty_params));- ASSERT_FALSE(auth_manager.authenticate("SXZqcVdOWjVNNUVsY3ZiTW9YajQ1QnhrUXJaRzRaS0VhTlFvUmlvQ3gycz1LZXlWeyJmaWx0ZXJfYnkiOiAidXNlcl9pZDoxMDgw In0=", "documents:search", {"coll1"}, empty_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", " XhsdBdhehdDheruyhvbdhwjhHdhgyeHbfheR")}, empty_params, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", "cXYPvkNKRlQrBzVTEgY4a3FrZfZ2MEs4kFJ6all3eldwM GhKZnRId3Y3TT1RZmxZeYJmaWx0ZXJfYnkiOkJ1aWQ6OElVm1lUVm15SG9ZOHM4NUx2VFk4S2drNHJIMiJ9")}, empty_params, embedded_params));+ ASSERT_FALSE(auth_manager.authenticate("documents:search", {collection_key_t("coll1", "SXZqcVdOWjVNNUVsY3ZiTW9YajQ1QnhrUXJaRzRaS0VhTlFvUmlvQ3gycz1LZXlWeyJmaWx0ZXJfYnkiOiAidXNlcl9pZDoxMDgw In0=")}, empty_params, embedded_params));}TEST_F(AuthManagerTest, ValidateBadKeyProperties) {diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 931ca6f5..1926d174 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -316,6 +316,66 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(4, results["hits"].size());}+TEST_F(CollectionManagerTest, VerifyEmbeddedParametersOfScopedAPIKey) {+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),+ field("year", field_types::INT32, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Tom Sawyer";+ doc1["year"] = 1876;+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Tom Sawyer";+ doc2["year"] = 1922;+ doc2["points"] = 200;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("*", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ std::map<std::string, std::string> req_params;+ req_params["collection"] = "coll1";+ req_params["q"] = "*";++ nlohmann::json embedded_params;+ embedded_params["filter_by"] = "points: 200";++ std::string json_res;+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res);+ ASSERT_TRUE(search_op.ok());++ nlohmann::json res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // existing filter should be augmented+ req_params.clear();+ req_params["collection"] = "coll1";+ req_params["filter_by"] = "year: 1922";+ req_params["q"] = "*";++ search_op = collectionManager.do_search(req_params, embedded_params, json_res);+ ASSERT_TRUE(search_op.ok());+ res_obj = nlohmann::json::parse(json_res);++ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ("year: 1922&&points: 200", req_params["filter_by"]);++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {Collection *coll1;diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 3dcadc68..e21e02ca 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -191,11 +191,14 @@ TEST_F(CoreAPIUtilsTest, MultiSearchEmbeddedKeys) {body["searches"].push_back(search);req->body = body.dump();+ nlohmann::json embedded_params;+ embedded_params["filter_by"] = "foo: bar";+ req->embedded_params_vec.push_back(embedded_params);post_multi_search(req, res);// ensure that req params are appended to (embedded params are also rolled into req params)- ASSERT_EQ("user_id: 100&&age: > 100", req->params["filter_by"]);+ ASSERT_EQ("user_id: 100&&age: > 100&&foo: bar", req->params["filter_by"]);}TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {@@ -212,11 +215,12 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {)";route_path rpath("POST", {"collections"}, post_create_collection, false, false);- std::vector<std::string> collections;+ std::vector<collection_key_t> collections;- get_collections_for_auth(req_params, body, rpath, collections);+ get_collections_for_auth(req_params, body, rpath, "foo", collections);ASSERT_EQ(1, collections.size());- ASSERT_STREQ("coll1", collections[0].c_str());+ ASSERT_EQ("coll1", collections[0].collection);+ ASSERT_EQ("foo", collections[0].api_key);// badly constructed collection schema bodycollections.clear();@@ -231,10 +235,13 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {})";- get_collections_for_auth(req_params, body, rpath, collections);- ASSERT_EQ(0, collections.size());+ get_collections_for_auth(req_params, body, rpath, "foo", collections);+ ASSERT_EQ(1, collections.size());+ ASSERT_EQ("", collections[0].collection);+ ASSERT_EQ("foo", collections[0].api_key);collections.clear();+ // missing collection namebody = R"({"fields": [@@ -245,8 +252,10 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {})";- get_collections_for_auth(req_params, body, rpath, collections);- ASSERT_EQ(0, collections.size());+ get_collections_for_auth(req_params, body, rpath, "foo", collections);+ ASSERT_EQ(1, collections.size());+ ASSERT_EQ("", collections[0].collection);+ ASSERT_EQ("foo", collections[0].api_key);// check for multi_searchcollections.clear();@@ -256,15 +265,17 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {{"query_by": "concat","collection": "products",- "q": "battery"+ "q": "battery",+ "x-typesense-api-key": "bar"}]})";- get_collections_for_auth(req_params, body, rpath, collections);+ get_collections_for_auth(req_params, body, rpath, "foo", collections);ASSERT_EQ(1, collections.size());- ASSERT_STREQ("products", collections[0].c_str());+ ASSERT_EQ("products", collections[0].collection);+ ASSERT_EQ("bar", collections[0].api_key);// get collection for multi-searchcollections.clear();@@ -279,32 +290,38 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBody) {})";- get_collections_for_auth(req_params, body, rpath, collections);- ASSERT_EQ(0, collections.size());+ get_collections_for_auth(req_params, body, rpath, "foo", collections);+ ASSERT_EQ(1, collections.size());+ ASSERT_EQ("", collections[0].collection);+ ASSERT_EQ("foo", collections[0].api_key);collections.clear();body = R"({"searches":[{"query_by": "concat",- "q": "battery"+ "q": "battery",+ "x-typesense-api-key": "bar"}]})";- get_collections_for_auth(req_params, body, rpath, collections);- ASSERT_EQ(0, collections.size());+ get_collections_for_auth(req_params, body, rpath, "foo", collections);+ ASSERT_EQ(1, collections.size());+ ASSERT_EQ("", collections[0].collection);+ ASSERT_EQ("bar", collections[0].api_key);}TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBodyExtended) {route_path rpath_multi_search = route_path("POST", {"multi_search"}, post_multi_search, false, false);std::map<std::string, std::string> req_params;- std::vector<std::string> collections;- get_collections_for_auth(req_params, "{]", rpath_multi_search, collections);+ std::vector<collection_key_t> collections;+ get_collections_for_auth(req_params, "{]", rpath_multi_search, "", collections);- ASSERT_EQ(0, collections.size());+ ASSERT_EQ(1, collections.size());+ ASSERT_EQ("", collections[0].collection);nlohmann::json sample_search_body;sample_search_body["searches"] = nlohmann::json::array();@@ -318,28 +335,38 @@ TEST_F(CoreAPIUtilsTest, ExtractCollectionsFromRequestBodyExtended) {sample_search_body["searches"].push_back(search_query);collections.clear();- get_collections_for_auth(req_params, sample_search_body.dump(), rpath_multi_search, collections);+ get_collections_for_auth(req_params, sample_search_body.dump(), rpath_multi_search, "", collections);ASSERT_EQ(2, collections.size());- ASSERT_STREQ("company1", collections[0].c_str());- ASSERT_STREQ("company2", collections[1].c_str());+ ASSERT_EQ("company1", collections[0].collection);+ ASSERT_EQ("company2", collections[1].collection);collections.clear();req_params["collection"] = "foo";- get_collections_for_auth(req_params, sample_search_body.dump(), rpath_multi_search, collections);+ get_collections_for_auth(req_params, sample_search_body.dump(), rpath_multi_search, "", collections);- ASSERT_EQ(3, collections.size());- ASSERT_STREQ("foo", collections[0].c_str());- ASSERT_STREQ("company1", collections[1].c_str());- ASSERT_STREQ("company2", collections[2].c_str());+ ASSERT_EQ(2, collections.size());+ ASSERT_EQ("company1", collections[0].collection);+ ASSERT_EQ("company2", collections[1].collection);++ collections.clear();++ // when one of the search arrays don't have an explicit collection, use the collection name from req param+ sample_search_body["searches"][1].erase("collection");++ get_collections_for_auth(req_params, sample_search_body.dump(), rpath_multi_search, "", collections);++ ASSERT_EQ(2, collections.size());+ ASSERT_EQ("company1", collections[0].collection);+ ASSERT_EQ("foo", collections[1].collection);collections.clear();req_params.clear();route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, get_search, false, false);- get_collections_for_auth(req_params, sample_search_body.dump(), rpath_search, collections);+ get_collections_for_auth(req_params, sample_search_body.dump(), rpath_search, "", collections);ASSERT_EQ(1, collections.size());- ASSERT_STREQ("", collections[0].c_str());+ ASSERT_EQ("", collections[0].collection);}