Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
return collections as per scoped key params (#1658)
Krunal Gandhic9618bf71ead
diff --git a/include/auth_manager.h b/include/auth_manager.hindex 9f155a5b..4a0c7080 100644--- a/include/auth_manager.h+++ b/include/auth_manager.h@@ -156,4 +156,6 @@ public:bool overwrite);void do_housekeeping();++ std::vector<std::string> get_api_key_collections(const std::string& value);};\ No newline at end of filediff --git a/include/collection_manager.h b/include/collection_manager.hindex 5cca9c09..106b3caf 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -137,7 +137,8 @@ public:void add_to_collections(Collection* collection);- Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0) const;+ Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0,+ const std::vector<std::string>& api_key_collections = {}) const;std::vector<std::string> get_collection_names() const;@@ -179,7 +180,8 @@ public:locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;- Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0) const;+ Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0,+ const std::vector<std::string>& api_key_collections = {}) const;Option<nlohmann::json> drop_collection(const std::string& collection_name,const bool remove_from_store = true,@@ -241,4 +243,6 @@ public:std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins);std::unordered_set<std::string> get_collection_references(const std::string& coll_name);++ bool is_valid_api_key_collection(const std::vector<std::string>& api_key_collections, Collection* coll) const;};diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex efefb343..ab5dede0 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -438,3 +438,11 @@ void AuthManager::remove_expired_keys() {void AuthManager::do_housekeeping() {remove_expired_keys();}++std::vector<std::string> AuthManager::get_api_key_collections(const std::string& value) {+ if(api_keys.find(value) != api_keys.end()) {+ return api_keys.at(value).collections;+ }++ return std::vector<std::string>();+}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex c5b37c74..e48266f2 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -581,7 +581,9 @@ locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uinreturn locked_resource_view_t<Collection>(mutex, nullptr);}-Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset) const {+Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset,+ const std::vector<std::string>& api_key_collections) const {+std::shared_lock lock(mutex);std::vector<Collection*> collection_vec;@@ -603,7 +605,9 @@ Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t lim}for (collections_it; collections_it != collections_end; ++collections_it) {- collection_vec.push_back(collections_it->second);+ if(is_valid_api_key_collection(api_key_collections, collections_it->second)) {+ collection_vec.push_back(collections_it->second);+ }}@@ -1956,10 +1960,11 @@ ThreadPool* CollectionManager::get_thread_pool() const {return thread_pool;}-Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset) const {+Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset,+ const std::vector<std::string>& api_key_collections) const {std::shared_lock lock(mutex);- auto collections_op = get_collections(limit, offset);+ auto collections_op = get_collections(limit, offset, api_key_collections);if(!collections_op.ok()) {return Option<nlohmann::json>(collections_op.code(), collections_op.error());}@@ -2457,3 +2462,17 @@ std::unordered_set<std::string> CollectionManager::get_collection_references(con}return references;}++bool CollectionManager::is_valid_api_key_collection(const std::vector<std::string>& api_collections, Collection* coll) const {+ for(const auto& api_collection : api_collections) {+ if(api_collection == "*") {+ return true;+ }++ const std::regex pattern(api_collection);+ if(std::regex_match(coll->get_name(), pattern)) {+ return true;+ }+ }+ return api_collections.size() > 0 ? false : true;+}\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex 67bb55dd..e35ea31d 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -208,7 +208,10 @@ bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptrlimit = std::stoi(limit_str);}- auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset);+ AuthManager &auth_manager = collectionManager.getAuthManager();+ auto api_key_collections = auth_manager.get_api_key_collections(req->api_auth_key);++ auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset, api_key_collections);if(!collections_summaries_op.ok()) {res->set(collections_summaries_op.code(), collections_summaries_op.error());return false;diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cppindex ee35d3e2..6f2ccf41 100644--- a/test/auth_manager_test.cpp+++ b/test/auth_manager_test.cpp@@ -4,6 +4,7 @@#include <http_data.h>#include "auth_manager.h"#include "core_api.h"+#include <collection_manager.h>static const size_t FUTURE_TS = 64723363199;@@ -11,6 +12,8 @@ class AuthManagerTest : public ::testing::Test {protected:Store *store;AuthManager auth_manager;+ CollectionManager& collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/auth_manager_test_db";@@ -18,6 +21,8 @@ protected:store = new Store(state_dir_path);auth_manager.init(store, "bootstrap-key");+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);}virtual void SetUp() {@@ -514,4 +519,90 @@ TEST_F(AuthManagerTest, AutoDeleteKeysOnExpiry) {keys = list_op.get();ASSERT_EQ(1, keys.size());ASSERT_EQ("abcd", keys[0].value);+}++TEST_F(AuthManagerTest, CollectionsByScope) {+ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ auto scoped_key_json = R"({+ "description": "Write key",+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],+ "collections": [ "collection_.*" ],+ "value": "3859c47b98"+ })"_json;++ req->body =scoped_key_json.dump();+ ASSERT_TRUE(post_create_key(req, res));++ auto schema1 = R"({+ "name": "collection_1",+ "fields": [+ {"name": "title", "type": "string", "locale": "en"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;++ collectionManager.create_collection(schema1);++ auto schema2 = R"({+ "name": "collection2",+ "fields": [+ {"name": "title", "type": "string", "locale": "en"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;++ collectionManager.create_collection(schema2);+++ req->api_auth_key = "3859c47b98";+ get_collections(req, res);+ auto result_json = nlohmann::json::parse(res->body);++ ASSERT_EQ(1, result_json.size());+ ASSERT_EQ("collection_1", result_json[0]["name"]);++ req->api_auth_key.clear();+ get_collections(req, res);+ result_json = nlohmann::json::parse(res->body);++ ASSERT_EQ(2, result_json.size());+ ASSERT_EQ("collection2", result_json[0]["name"]);+ ASSERT_EQ("collection_1", result_json[1]["name"]);++ scoped_key_json = R"({+ "description": "Write key",+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],+ "collections": [ "collection2" ],+ "value": "b78a573a1a"+ })"_json;++ req->body =scoped_key_json.dump();+ ASSERT_TRUE(post_create_key(req, res));++ req->api_auth_key = "b78a573a1a";+ get_collections(req, res);+ result_json = nlohmann::json::parse(res->body);++ ASSERT_EQ(1, result_json.size());+ ASSERT_EQ("collection2", result_json[0]["name"]);++ scoped_key_json = R"({+ "description": "Write key",+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],+ "collections": [ "*" ],+ "value": "00071e2108"+ })"_json;++ req->body =scoped_key_json.dump();+ ASSERT_TRUE(post_create_key(req, res));++ req->api_auth_key = "00071e2108";+ get_collections(req, res);+ result_json = nlohmann::json::parse(res->body);++ ASSERT_EQ(2, result_json.size());+ ASSERT_EQ("collection2", result_json[0]["name"]);+ ASSERT_EQ("collection_1", result_json[1]["name"]);}\ No newline at end of filediff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 0a855e3e..68ab3fa7 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -2251,9 +2251,12 @@ TEST_F(CollectionFacetingTest, FacetingReturnParent) {ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);ASSERT_EQ(1, results["facet_counts"][2]["counts"].size());- ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][2]["counts"][0]["parent"].dump());ASSERT_EQ("0", results["facet_counts"][2]["counts"][0]["value"]);+ //same facet value appearing in multiple records can return any parent+ ASSERT_TRUE(("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}" == results["facet_counts"][2]["counts"][0]["parent"].dump())+ || ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}" == results["facet_counts"][2]["counts"][0]["parent"].dump()));+ASSERT_EQ(2, results["facet_counts"][3]["counts"].size());ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][3]["counts"][0]["parent"].dump());ASSERT_EQ("0", results["facet_counts"][3]["counts"][0]["value"]);diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cppindex 634335a1..546112f5 100644--- a/test/collection_optimized_faceting_test.cpp+++ b/test/collection_optimized_faceting_test.cpp@@ -1806,6 +1806,52 @@ TEST_F(CollectionOptimizedFacetingTest, FacetingReturnParent) {ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);++ //return parent for multiple facet fields+ search_op = coll1->search("*", {},"", {"value.color", "value.r", "value.g", "value.b"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, VALUE, 30000,+ 2, "", {"value.r", "value.g", "value.b"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();+ ASSERT_EQ(4, results["facet_counts"].size());++ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][0]["parent"].dump());+ ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());+ ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);++ ASSERT_EQ(1, results["facet_counts"][2]["counts"].size());+ ASSERT_EQ("0", results["facet_counts"][2]["counts"][0]["value"]);++ //same facet value appearing in multiple records can return any parent+ ASSERT_TRUE(("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}" == results["facet_counts"][2]["counts"][0]["parent"].dump())+ || ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}" == results["facet_counts"][2]["counts"][0]["parent"].dump()));++ ASSERT_EQ(2, results["facet_counts"][3]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][3]["counts"][0]["parent"].dump());+ ASSERT_EQ("0", results["facet_counts"][3]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][3]["counts"][1]["parent"].dump());+ ASSERT_EQ("255", results["facet_counts"][3]["counts"][1]["value"]);}TEST_F(CollectionOptimizedFacetingTest, FacetingReturnParentDeepNested) {