Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Manage scoped search API keys.
kishorenc1395d74d488d
diff --git a/include/auth_manager.h b/include/auth_manager.hindex 7db4d199..fceee6e1 100644--- a/include/auth_manager.h+++ b/include/auth_manager.h@@ -14,6 +14,8 @@ struct api_key_t {std::vector<std::string> actions;std::vector<std::string> collections;+ static const size_t PREFIX_LEN = 4;+api_key_t() {}@@ -68,7 +70,7 @@ struct api_key_t {}api_key_t& truncate_value() {- value = value.substr(0, 4); // return only first 4 chars+ value = value.substr(0, PREFIX_LEN); // return only first 4 charsreturn (*this);}};@@ -89,6 +91,8 @@ private:uint32_t get_next_api_key_id();+ static constexpr const char* DOCUMENTS_SEARCH_ACTION = "documents:search";+public:AuthManager() = default;@@ -106,6 +110,16 @@ public:bool authenticate(const std::string& req_api_key,const std::string& action,+ const std::string& collection,+ std::map<std::string, std::string> & params+ );++ Option<std::string> params_from_scoped_key(+ const std::string& scoped_api_key,+ const std::string& action,const std::string& collection);++ static const size_t KEY_LEN = 32;+ static const size_t HMAC_BASE64_LEN = 44;};\ No newline at end of filediff --git a/include/collection_manager.h b/include/collection_manager.hindex 5c6d34a1..49f5f0c0 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -73,7 +73,8 @@ public:void add_to_collections(Collection* collection);- bool auth_key_matches(const std::string& auth_key_sent, const std::string& action, const std::string& collection);+ bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,+ const std::string& collection, std::map<std::string, std::string> & params);Option<Collection*> create_collection(const std::string name, const std::vector<field> & fields,const std::string & default_sorting_field,diff --git a/include/core_api.h b/include/core_api.hindex 790d812b..f2c227ea 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -2,7 +2,7 @@#include "http_server.h"-bool handle_authentication(const http_req& req, const route_path& rpath, const std::string& auth_key);+bool handle_authentication(http_req& req, const route_path& rpath, const std::string& auth_key);bool get_collections(http_req & req, http_res & res);diff --git a/include/http_data.h b/include/http_data.hindex 277814e5..b69fa815 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -223,10 +223,11 @@ struct route_path {// e.g. /collections/:collection/documents/searchoperation = path_parts[resource_index+1];} else {- // e.g /collections or /collections/:collection/documents or /collections/:collection- // get will be a list, post would be a create, etc.+ // e.g /collections or /collections/:collection/foo or /collections/:collection+if(http_method == "GET") {- operation = "list";+ // GET can be a `get` or `list`+ operation = (resource_index == path_parts.size()-1) ? "list" : "get";} else if(http_method == "POST") {operation = "create";} else if(http_method == "PUT") {diff --git a/include/http_server.h b/include/http_server.hindex d0bd5bcd..ac825a3b 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -61,7 +61,7 @@ private:bool cors_enabled;- bool (*auth_handler)(const http_req& req, const route_path& rpath, const std::string& auth_key);+ bool (*auth_handler)(http_req& req, const route_path& rpath, const std::string& auth_key);static void on_accept(h2o_socket_t *listener, const char *err);@@ -93,7 +93,7 @@ public:bool is_alive() const;- void set_auth_handler(bool (*handler)(const http_req & req, const route_path & rpath,+ void set_auth_handler(bool (*handler)(http_req & req, const route_path & rpath,const std::string & auth_key));void get(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);diff --git a/include/index.h b/include/index.hindex 5d088853..9f2d2edd 100644--- a/include/index.h+++ b/include/index.h@@ -129,8 +129,10 @@ private:spp::sparse_hash_map<std::string, art_tree*> search_index;+ // seq_id => (facet => values)spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;+ // sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;StringUtils string_utils;diff --git a/include/string_utils.h b/include/string_utils.hindex 13cda34f..844ee383 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -80,6 +80,9 @@ struct StringUtils {return isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10;}+ // Convert string of chars to its representative string of hex numbers+ static std::string str2hex(const std::string& str, bool capital = false);+static std::string url_decode(std::string text) {char h;std::ostringstream escaped;@@ -228,4 +231,6 @@ struct StringUtils {}static std::string randstring(size_t length, uint64_t seed);++ static std::string hmac(const std::string& key, const std::string& msg);};\ No newline at end of filediff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex e9f04ed5..9374cf6b 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -1,4 +1,8 @@#include "auth_manager.h"+#include <openssl/evp.h>+#include <openssl/hmac.h>++constexpr const char* AuthManager::DOCUMENTS_SEARCH_ACTION;Option<std::vector<api_key_t>> AuthManager::list_keys() {std::vector<std::string> api_key_json_strs;@@ -119,7 +123,44 @@ Option<bool> AuthManager::init(Store *store) {}bool AuthManager::authenticate(const std::string& req_api_key, const std::string& action,- const std::string& collection) {+ const std::string& collection, std::map<std::string, std::string> & params) {++ if(req_api_key.size() > KEY_LEN) {+ // scoped API key+ Option<std::string> params_op = params_from_scoped_key(req_api_key, action, collection);+ if(!params_op.ok()) {+ return false;+ }++ const std::string& embedded_params_str = params_op.get();+ nlohmann::json embedded_params;++ try {+ embedded_params = nlohmann::json::parse(embedded_params_str);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ return false;+ }++ if(!embedded_params.is_object()) {+ LOG(ERROR) << "Scoped API key contains invalid search parameters.";+ return false;+ }++ // enrich params with values from embedded_params+ for (const auto& it: embedded_params.items()){+ if(params.count(it.key()) == 0) {+ params[it.key()] = it.value();+ } else if(it.key() == "filter_by") {+ params[it.key()] = params[it.key()] + "&&" + it.value().get<std::string>();+ } else {+ params[it.key()] = it.value();+ }+ }++ return true;+ }+if(api_keys.count(req_api_key) == 0) {return false;}@@ -151,3 +192,55 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::stringreturn false;}+Option<std::string> AuthManager::params_from_scoped_key(const std::string &scoped_api_key, const std::string& action,+ const std::string& collection) {+ // allow only searches from scoped keys+ if(action != DOCUMENTS_SEARCH_ACTION) {+ LOG(ERROR) << "Scoped API keys can only be used for searches.";+ return Option<std::string>(403, "Forbidden.");+ }++ const std::string& key_payload = StringUtils::base64_decode(scoped_api_key);++ // FORMAT:+ // <DIGEST><PARENT_KEY_PREFIX><PARAMS>+ const std::string& hmacSHA256 = key_payload.substr(0, HMAC_BASE64_LEN);+ const std::string& key_prefix = key_payload.substr(HMAC_BASE64_LEN, api_key_t::PREFIX_LEN);+ const std::string& custom_params = key_payload.substr(HMAC_BASE64_LEN + api_key_t::PREFIX_LEN);++ // calculate and verify hmac against matching api key+ for (const auto &kv : api_keys) {+ if(kv.first.substr(0, api_key_t::PREFIX_LEN) == key_prefix) {+ const api_key_t& api_key = kv.second;++ // ensure that parent key has only search scope+ if(api_key.actions.size() != 1 || api_key.actions[0] != DOCUMENTS_SEARCH_ACTION) {+ LOG(ERROR) << "Parent API key must allow only `" << DOCUMENTS_SEARCH_ACTION << "` action.";+ return Option<std::string>(403, "Forbidden.");+ }++ // ensure that parent key collection filter matches queried collection+ bool collection_allowed = false;+ for(const std::string& allowed_collection: api_key.collections) {+ if(allowed_collection == "*" || (collection != "*" && allowed_collection == collection)) {+ collection_allowed = true;+ break;+ }+ }++ if(!collection_allowed) {+ LOG(ERROR) << "Parent key does not allow queries against queried collection.";+ return Option<std::string>(403, "Forbidden.");+ }++ // finally verify hmac+ std::string digest = StringUtils::hmac(kv.first, custom_params);++ if(digest == hmacSHA256) {+ return Option<std::string>(custom_params);+ }+ }+ }++ return Option<std::string>(403, "Forbidden.");+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex f2841513..adf60e8a 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -208,8 +208,10 @@ void CollectionManager::dispose() {store->close();}-bool CollectionManager::auth_key_matches(const std::string& auth_key_sent, const std::string& action,- const std::string& collection) {+bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,+ const std::string& action,+ const std::string& collection,+ std::map<std::string, std::string> & params) {if(auth_key_sent.empty()) {return false;}@@ -225,7 +227,7 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent, const}// finally, check managed auth keys- return auth_manager.authenticate(auth_key_sent, action, collection);+ return auth_manager.authenticate(auth_key_sent, action, collection, params);}Option<Collection*> CollectionManager::create_collection(const std::string name, const std::vector<field> & fields,diff --git a/src/core_api.cpp b/src/core_api.cppindex 1e2ee439..999f65ff 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -32,7 +32,7 @@ nlohmann::json collection_summary_json(Collection *collection) {return json_response;}-bool handle_authentication(const http_req& req, const route_path& rpath, const std::string& auth_key) {+bool handle_authentication(http_req& req, const route_path& rpath, const std::string& auth_key) {CollectionManager & collectionManager = CollectionManager::get_instance();std::string collection = "*";@@ -46,7 +46,7 @@ bool handle_authentication(const http_req& req, const route_path& rpath, const sreturn true;}- return collectionManager.auth_key_matches(auth_key, rpath.action, collection);+ return collectionManager.auth_key_matches(auth_key, rpath.action, collection, req.params);}bool get_collections(http_req & req, http_res & res) {diff --git a/src/http_server.cpp b/src/http_server.cppindex 7e695789..e5d09a30 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -236,8 +236,10 @@ std::map<std::string, std::string> HttpServer::parse_query(const std::string& qustd::string value = StringUtils::url_decode(raw_value);if(query_map.count(key) == 0) {query_map[key] = value;- } else {+ } else if(key == "filter_by") {query_map[key] = query_map[key] + "&&" + value;+ } else {+ query_map[key] = value;}}@@ -421,7 +423,7 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::stringreturn 0;}-void HttpServer::set_auth_handler(bool (*handler)(const http_req& req, const route_path& rpath,+void HttpServer::set_auth_handler(bool (*handler)(http_req& req, const route_path& rpath,const std::string& auth_key)) {auth_handler = handler;}diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 6e1d85c9..2782c2b6 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -1,5 +1,8 @@#include "string_utils.h"#include <iostream>+#include <openssl/evp.h>+#include <iomanip>+#include <openssl/hmac.h>std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;@@ -66,3 +69,26 @@ std::string StringUtils::randstring(size_t length, uint64_t seed) {return s;}++std::string StringUtils::hmac(const std::string& key, const std::string& msg) {+ unsigned int hmac_len;+ unsigned char hmac[EVP_MAX_MD_SIZE];+ HMAC(EVP_sha256(), key.c_str(), key.size(),+ reinterpret_cast<const unsigned char *>(msg.c_str()), msg.size(),+ hmac, &hmac_len);++ std::string digest_raw(reinterpret_cast<char*>(&hmac), hmac_len);+ return StringUtils::base64_encode(digest_raw);+}++std::string StringUtils::str2hex(const std::string &str, bool capital) {+ std::string hexstr;+ hexstr.resize(str.size() * 2);+ const size_t a = capital ? 'A' - 1 : 'a' - 1;+ for (size_t i = 0, c = str[0] & 0xFF; i < hexstr.size(); c = str[i / 2] & 0xFF) {+ hexstr[i++] = c > 0x9F ? (c / 16 - 9) | a : c / 16 | '0';+ hexstr[i++] = (c & 0xF) > 9 ? (c % 16 - 9) | a : c % 16 | '0';+ }++ return hexstr;+}diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cppindex 402e3051..73cf6afe 100644--- a/test/auth_manager_test.cpp+++ b/test/auth_manager_test.cpp@@ -1,6 +1,7 @@#include <gtest/gtest.h>#include <stdlib.h>#include <iostream>+#include <http_data.h>#include "auth_manager.h"class AuthManagerTest : public ::testing::Test {@@ -110,52 +111,112 @@ TEST_F(AuthManagerTest, CheckRestoreOfAPIKeys) {}TEST_F(AuthManagerTest, VerifyAuthentication) {+ std::map<std::string, std::string> sparams;// when no keys are present at all- ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "", ""));+ ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "", "", sparams));// wildcard permissionapi_key_t wildcard_all_key = api_key_t("abcd1", "wildcard all key", {"*"}, {"*"});auth_manager.create_key(wildcard_all_key);- ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "documents:create", "collection1"));- ASSERT_TRUE(auth_manager.authenticate(wildcard_all_key.value, "metrics:get", ""));+ ASSERT_FALSE(auth_manager.authenticate("jdlaslasdasd", "documents:create", "collection1", sparams));+ ASSERT_TRUE(auth_manager.authenticate(wildcard_all_key.value, "metrics:get", "", sparams));// wildcard on a collectionapi_key_t wildcard_coll_key = api_key_t("abcd2", "wildcard coll key", {"*"}, {"collection1"});auth_manager.create_key(wildcard_coll_key);- ASSERT_FALSE(auth_manager.authenticate("adasda", "documents:create", "collection1"));- ASSERT_TRUE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", "collection1"));- ASSERT_FALSE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", "collection2"));+ ASSERT_FALSE(auth_manager.authenticate("adasda", "documents:create", "collection1", sparams));+ ASSERT_TRUE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", "collection1", sparams));+ ASSERT_FALSE(auth_manager.authenticate(wildcard_coll_key.value, "documents:get", "collection2", sparams));// wildcard on multiple collectionsapi_key_t wildcard_colls_key = api_key_t("abcd3", "wildcard coll key", {"*"}, {"collection1", "collection2", "collection3"});auth_manager.create_key(wildcard_colls_key);- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "collection1"));- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:search", "collection2"));- ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:create", "collection3"));- ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "collection4"));- ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "*"));+ ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "collection1", sparams));+ ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:search", "collection2", sparams));+ ASSERT_TRUE(auth_manager.authenticate(wildcard_colls_key.value, "documents:create", "collection3", sparams));+ ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "collection4", sparams));+ ASSERT_FALSE(auth_manager.authenticate(wildcard_colls_key.value, "documents:get", "*", sparams));// only 1 action on multiple collectionsapi_key_t one_action_key = api_key_t("abcd4", "one action key", {"documents:search"}, {"collection1", "collection2"});auth_manager.create_key(one_action_key);- ASSERT_TRUE(auth_manager.authenticate(one_action_key.value, "documents:search", "collection1"));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:get", "collection2"));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:search", "collection5"));- ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "*", "collection2"));+ ASSERT_TRUE(auth_manager.authenticate(one_action_key.value, "documents:search", "collection1", sparams));+ ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:get", "collection2", sparams));+ ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "documents:search", "collection5", sparams));+ ASSERT_FALSE(auth_manager.authenticate(one_action_key.value, "*", "collection2", sparams));// multiple actions on multiple collectionsapi_key_t mul_acoll_key = api_key_t("abcd5", "multiple action/collection key",{"documents:get", "collections:list"}, {"metacollection", "collection2"});auth_manager.create_key(mul_acoll_key);- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", "metacollection"));- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", "collection2"));- ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", "metacollection"));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:search", "collection2"));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", "collection5"));- ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "*", "*"));-}\ No newline at end of file+ ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", "metacollection", sparams));+ ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", "collection2", sparams));+ ASSERT_TRUE(auth_manager.authenticate(mul_acoll_key.value, "collections:list", "metacollection", sparams));+ ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:search", "collection2", sparams));+ ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "documents:get", "collection5", sparams));+ ASSERT_FALSE(auth_manager.authenticate(mul_acoll_key.value, "*", "*", sparams));+}++TEST_F(AuthManagerTest, GenerationOfAPIAction) {+ route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false);+ route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false);+ route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false);+ route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false);+ route_path rpath_doc_delete = route_path("DELETE", {"collections", ":collection", "documents", ":id"}, nullptr, false);+ route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false);++ ASSERT_STREQ("documents:search", rpath_search._get_action().c_str());+ ASSERT_STREQ("collections:get", rpath_coll_get._get_action().c_str());+ ASSERT_STREQ("collections:list", rpath_coll_list._get_action().c_str());+ ASSERT_STREQ("keys:create", rpath_keys_post._get_action().c_str());+ ASSERT_STREQ("documents:delete", rpath_doc_delete._get_action().c_str());+ ASSERT_STREQ("overrides:upsert", rpath_override_upsert._get_action().c_str());+}++TEST_F(AuthManagerTest, ScopedAPIKeys) {+ std::map<std::string, std::string> params;+ params["filter_by"] = "country:USA";++ // create a API key bound to search scope and a given collection+ api_key_t key_search_coll1("KeyVal", "test key", {"documents:search"}, {"coll1"});+ auth_manager.create_key(key_search_coll1);++ std::string scoped_key = StringUtils::base64_encode(+ "IvjqWNZ5M5ElcvbMoXj45BxkQrZG4ZKEaNQoRioCx2s=KeyV{\"filter_by\": \"user_id:1080\"}"+ );++ ASSERT_TRUE(auth_manager.authenticate(scoped_key, "documents:search", "coll1", params));+ ASSERT_STREQ("country:USA&&user_id:1080", params["filter_by"].c_str());++ // should scope to collection bound by the parent key+ ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:search", "coll2", params));++ // should scope to search action only+ ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:create", "coll1", params));++ // check with corrupted key+ ASSERT_FALSE(auth_manager.authenticate("asdasasd", "documents:search", "coll1", params));++ // when params is empty, embedded param should be set+ std::map<std::string, std::string> empty_params;+ ASSERT_TRUE(auth_manager.authenticate(scoped_key, "documents:search", "coll1", empty_params));+ ASSERT_STREQ("user_id:1080", empty_params["filter_by"].c_str());++ // when more than a single key prefix matches, must pick the correct underlying key+ api_key_t key_search_coll2("KeyVal2", "test key", {"documents:search"}, {"coll2"});+ auth_manager.create_key(key_search_coll2);+ ASSERT_FALSE(auth_manager.authenticate(scoped_key, "documents:search", "coll2", empty_params));++ // should only allow scoped API keys derived from parent key with documents:search action+ api_key_t key_search_admin("AdminKey", "admin key", {"*"}, {"*"});+ auth_manager.create_key(key_search_admin);+ std::string scoped_key2 = StringUtils::base64_encode(+ "BXbsk+xLT1gxOjDyip6+PE4MtOzOm/H7kbkN1d/j/s4=Admi{\"filter_by\": \"user_id:1080\"}"+ );+ ASSERT_FALSE(auth_manager.authenticate(scoped_key2, "documents:search", "coll", empty_params));+}diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 4aa63c61..4f8c2400 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -49,3 +49,8 @@ TEST(StringUtilsTest, ShouldJoinString) {const std::string & joined_str3 = StringUtils::join({}, "/");ASSERT_STREQ("", joined_str3.c_str());}++TEST(StringUtilsTest, HMAC) {+ std::string digest1 = StringUtils::hmac("KeyVal", "{\"filter_by\": \"user_id:1080\"}");+ ASSERT_STREQ("IvjqWNZ5M5ElcvbMoXj45BxkQrZG4ZKEaNQoRioCx2s=", digest1.c_str());+}