Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

return collections as per scoped key params (#1658)

Krunal Gandhic9618bf71ead
diff --git a/include/auth_manager.h b/include/auth_manager.h
index 9f155a5b..4a0c7080 100644
--- a/include/auth_manager.h
+++ b/include/auth_manager.h
@@ -156,4 +156,6 @@ public:
bool overwrite);
void do_housekeeping();
+
+ std::vector<std::string> get_api_key_collections(const std::string& value);
};
\ No newline at end of file
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 5cca9c09..106b3caf 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -137,7 +137,8 @@ public:
void add_to_collections(Collection* collection);
- Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0) const;
+ Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0,
+ const std::vector<std::string>& api_key_collections = {}) const;
std::vector<std::string> get_collection_names() const;
@@ -179,7 +180,8 @@ public:
locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;
- Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0) const;
+ Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0,
+ const std::vector<std::string>& api_key_collections = {}) const;
Option<nlohmann::json> drop_collection(const std::string& collection_name,
const bool remove_from_store = true,
@@ -241,4 +243,6 @@ public:
std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins);
std::unordered_set<std::string> get_collection_references(const std::string& coll_name);
+
+ bool is_valid_api_key_collection(const std::vector<std::string>& api_key_collections, Collection* coll) const;
};
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index efefb343..ab5dede0 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -438,3 +438,11 @@ void AuthManager::remove_expired_keys() {
void AuthManager::do_housekeeping() {
remove_expired_keys();
}
+
+std::vector<std::string> AuthManager::get_api_key_collections(const std::string& value) {
+ if(api_keys.find(value) != api_keys.end()) {
+ return api_keys.at(value).collections;
+ }
+
+ return std::vector<std::string>();
+}
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index c5b37c74..e48266f2 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -581,7 +581,9 @@ locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uin
return locked_resource_view_t<Collection>(mutex, nullptr);
}
-Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset) const {
+Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset,
+ const std::vector<std::string>& api_key_collections) const {
+
std::shared_lock lock(mutex);
std::vector<Collection*> collection_vec;
@@ -603,7 +605,9 @@ Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t lim
}
for (collections_it; collections_it != collections_end; ++collections_it) {
- collection_vec.push_back(collections_it->second);
+ if(is_valid_api_key_collection(api_key_collections, collections_it->second)) {
+ collection_vec.push_back(collections_it->second);
+ }
}
@@ -1956,10 +1960,11 @@ ThreadPool* CollectionManager::get_thread_pool() const {
return thread_pool;
}
-Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset) const {
+Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset,
+ const std::vector<std::string>& api_key_collections) const {
std::shared_lock lock(mutex);
- auto collections_op = get_collections(limit, offset);
+ auto collections_op = get_collections(limit, offset, api_key_collections);
if(!collections_op.ok()) {
return Option<nlohmann::json>(collections_op.code(), collections_op.error());
}
@@ -2457,3 +2462,17 @@ std::unordered_set<std::string> CollectionManager::get_collection_references(con
}
return references;
}
+
+bool CollectionManager::is_valid_api_key_collection(const std::vector<std::string>& api_collections, Collection* coll) const {
+ for(const auto& api_collection : api_collections) {
+ if(api_collection == "*") {
+ return true;
+ }
+
+ const std::regex pattern(api_collection);
+ if(std::regex_match(coll->get_name(), pattern)) {
+ return true;
+ }
+ }
+ return api_collections.size() > 0 ? false : true;
+}
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 67bb55dd..e35ea31d 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -208,7 +208,10 @@ bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptr
limit = std::stoi(limit_str);
}
- auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset);
+ AuthManager &auth_manager = collectionManager.getAuthManager();
+ auto api_key_collections = auth_manager.get_api_key_collections(req->api_auth_key);
+
+ auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset, api_key_collections);
if(!collections_summaries_op.ok()) {
res->set(collections_summaries_op.code(), collections_summaries_op.error());
return false;
diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cpp
index ee35d3e2..6f2ccf41 100644
--- a/test/auth_manager_test.cpp
+++ b/test/auth_manager_test.cpp
@@ -4,6 +4,7 @@
#include <http_data.h>
#include "auth_manager.h"
#include "core_api.h"
+#include <collection_manager.h>
static const size_t FUTURE_TS = 64723363199;
@@ -11,6 +12,8 @@ class AuthManagerTest : public ::testing::Test {
protected:
Store *store;
AuthManager auth_manager;
+ CollectionManager& collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/auth_manager_test_db";
@@ -18,6 +21,8 @@ protected:
store = new Store(state_dir_path);
auth_manager.init(store, "bootstrap-key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
}
virtual void SetUp() {
@@ -514,4 +519,90 @@ TEST_F(AuthManagerTest, AutoDeleteKeysOnExpiry) {
keys = list_op.get();
ASSERT_EQ(1, keys.size());
ASSERT_EQ("abcd", keys[0].value);
+}
+
+TEST_F(AuthManagerTest, CollectionsByScope) {
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ auto scoped_key_json = R"({
+ "description": "Write key",
+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],
+ "collections": [ "collection_.*" ],
+ "value": "3859c47b98"
+ })"_json;
+
+ req->body =scoped_key_json.dump();
+ ASSERT_TRUE(post_create_key(req, res));
+
+ auto schema1 = R"({
+ "name": "collection_1",
+ "fields": [
+ {"name": "title", "type": "string", "locale": "en"},
+ {"name": "points", "type": "int32"}
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema1);
+
+ auto schema2 = R"({
+ "name": "collection2",
+ "fields": [
+ {"name": "title", "type": "string", "locale": "en"},
+ {"name": "points", "type": "int32"}
+ ]
+ })"_json;
+
+ collectionManager.create_collection(schema2);
+
+
+ req->api_auth_key = "3859c47b98";
+ get_collections(req, res);
+ auto result_json = nlohmann::json::parse(res->body);
+
+ ASSERT_EQ(1, result_json.size());
+ ASSERT_EQ("collection_1", result_json[0]["name"]);
+
+ req->api_auth_key.clear();
+ get_collections(req, res);
+ result_json = nlohmann::json::parse(res->body);
+
+ ASSERT_EQ(2, result_json.size());
+ ASSERT_EQ("collection2", result_json[0]["name"]);
+ ASSERT_EQ("collection_1", result_json[1]["name"]);
+
+ scoped_key_json = R"({
+ "description": "Write key",
+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],
+ "collections": [ "collection2" ],
+ "value": "b78a573a1a"
+ })"_json;
+
+ req->body =scoped_key_json.dump();
+ ASSERT_TRUE(post_create_key(req, res));
+
+ req->api_auth_key = "b78a573a1a";
+ get_collections(req, res);
+ result_json = nlohmann::json::parse(res->body);
+
+ ASSERT_EQ(1, result_json.size());
+ ASSERT_EQ("collection2", result_json[0]["name"]);
+
+ scoped_key_json = R"({
+ "description": "Write key",
+ "actions": [ "collections:*", "documents:*", "synonyms:*" ],
+ "collections": [ "*" ],
+ "value": "00071e2108"
+ })"_json;
+
+ req->body =scoped_key_json.dump();
+ ASSERT_TRUE(post_create_key(req, res));
+
+ req->api_auth_key = "00071e2108";
+ get_collections(req, res);
+ result_json = nlohmann::json::parse(res->body);
+
+ ASSERT_EQ(2, result_json.size());
+ ASSERT_EQ("collection2", result_json[0]["name"]);
+ ASSERT_EQ("collection_1", result_json[1]["name"]);
}
\ No newline at end of file
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 0a855e3e..68ab3fa7 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -2251,9 +2251,12 @@ TEST_F(CollectionFacetingTest, FacetingReturnParent) {
ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);
ASSERT_EQ(1, results["facet_counts"][2]["counts"].size());
- ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][2]["counts"][0]["parent"].dump());
ASSERT_EQ("0", results["facet_counts"][2]["counts"][0]["value"]);
+ //same facet value appearing in multiple records can return any parent
+ ASSERT_TRUE(("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}" == results["facet_counts"][2]["counts"][0]["parent"].dump())
+ || ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}" == results["facet_counts"][2]["counts"][0]["parent"].dump()));
+
ASSERT_EQ(2, results["facet_counts"][3]["counts"].size());
ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][3]["counts"][0]["parent"].dump());
ASSERT_EQ("0", results["facet_counts"][3]["counts"][0]["value"]);
diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cpp
index 634335a1..546112f5 100644
--- a/test/collection_optimized_faceting_test.cpp
+++ b/test/collection_optimized_faceting_test.cpp
@@ -1806,6 +1806,52 @@ TEST_F(CollectionOptimizedFacetingTest, FacetingReturnParent) {
ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);
ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());
ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);
+
+ //return parent for multiple facet fields
+ search_op = coll1->search("*", {},"", {"value.color", "value.r", "value.g", "value.b"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, VALUE, 30000,
+ 2, "", {"value.r", "value.g", "value.b"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+ ASSERT_EQ(4, results["facet_counts"].size());
+
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][0]["parent"].dump());
+ ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());
+ ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);
+
+ ASSERT_EQ(1, results["facet_counts"][2]["counts"].size());
+ ASSERT_EQ("0", results["facet_counts"][2]["counts"][0]["value"]);
+
+ //same facet value appearing in multiple records can return any parent
+ ASSERT_TRUE(("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}" == results["facet_counts"][2]["counts"][0]["parent"].dump())
+ || ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}" == results["facet_counts"][2]["counts"][0]["parent"].dump()));
+
+ ASSERT_EQ(2, results["facet_counts"][3]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][3]["counts"][0]["parent"].dump());
+ ASSERT_EQ("0", results["facet_counts"][3]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][3]["counts"][1]["parent"].dump());
+ ASSERT_EQ("255", results["facet_counts"][3]["counts"][1]["value"]);
}
TEST_F(CollectionOptimizedFacetingTest, FacetingReturnParentDeepNested) {