Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

pagination for collections GET (#1581)

Krunal Gandhi482858d05dc1
diff --git a/include/collection_manager.h b/include/collection_manager.h
index a88e3f7b..3473daa2 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -136,7 +136,7 @@ public:
void add_to_collections(Collection* collection);
- std::vector<Collection*> get_collections() const;
+ Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0) const;
std::vector<std::string> get_collection_names() const;
@@ -176,7 +176,7 @@ public:
locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;
- nlohmann::json get_collection_summaries() const;
+ Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0) const;
Option<nlohmann::json> drop_collection(const std::string& collection_name,
const bool remove_from_store = true,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 7f1a8aa2..464dc1ea 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -572,20 +572,40 @@ locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uin
return locked_resource_view_t<Collection>(mutex, nullptr);
}
-std::vector<Collection*> CollectionManager::get_collections() const {
+Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset) const {
std::shared_lock lock(mutex);
std::vector<Collection*> collection_vec;
- for(const auto& kv: collections) {
- collection_vec.push_back(kv.second);
+ auto collections_it = collections.begin();
+
+ if(offset > 0) {
+ if(offset >= collections.size()) {
+ return Option<std::vector<Collection*>>(400, "Invalid offset param.");
+ }
+
+ std::advance(collections_it, offset);
}
- std::sort(std::begin(collection_vec), std::end(collection_vec),
- [] (Collection* lhs, Collection* rhs) {
- return lhs->get_collection_id() > rhs->get_collection_id();
- });
+ auto collections_end = collections.end();
- return collection_vec;
+ if(limit > 0 && (offset + limit < collections.size())) {
+ collections_end = collections_it;
+ std::advance(collections_end, limit);
+ }
+
+ for (collections_it; collections_it != collections_end; ++collections_it) {
+ collection_vec.push_back(collections_it->second);
+ }
+
+
+ if(offset == 0 && limit == 0) { //dont sort for paginated requests
+ std::sort(std::begin(collection_vec), std::end(collection_vec),
+ [](Collection *lhs, Collection *rhs) {
+ return lhs->get_collection_id() > rhs->get_collection_id();
+ });
+ }
+
+ return Option<std::vector<Collection*>>(collection_vec);
}
std::vector<std::string> CollectionManager::get_collection_names() const {
@@ -1872,10 +1892,16 @@ ThreadPool* CollectionManager::get_thread_pool() const {
return thread_pool;
}
-nlohmann::json CollectionManager::get_collection_summaries() const {
+Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset) const {
std::shared_lock lock(mutex);
- std::vector<Collection*> colls = get_collections();
+ auto collections_op = get_collections(limit, offset);
+ if(!collections_op.ok()) {
+ return Option<nlohmann::json>(collections_op.code(), collections_op.error());
+ }
+
+ std::vector<Collection*> colls = collections_op.get();
+
nlohmann::json json_summaries = nlohmann::json::array();
for(Collection* collection: colls) {
@@ -1883,7 +1909,7 @@ nlohmann::json CollectionManager::get_collection_summaries() const {
json_summaries.push_back(collection_json);
}
- return json_summaries;
+ return Option<nlohmann::json>(json_summaries);
}
Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_json) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 60926709..d54ccf60 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -181,7 +181,33 @@ index_operation_t get_index_operation(const std::string& action) {
bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- nlohmann::json json_response = collectionManager.get_collection_summaries();
+
+ uint32_t offset = 0, limit = 0;
+ if(req->params.count("offset") != 0) {
+ const auto &offset_str = req->params["offset"];
+ if(!StringUtils::is_uint32_t(offset_str)) {
+ res->set(400, "Offset param should be unsigned integer.");
+ return false;
+ }
+ offset = std::stoi(offset_str);
+ }
+
+ if(req->params.count("limit") != 0) {
+ const auto &limit_str = req->params["limit"];
+ if(!StringUtils::is_uint32_t(limit_str)) {
+ res->set(400, "Limit param should be unsigned integer.");
+ return false;
+ }
+ limit = std::stoi(limit_str);
+ }
+
+ auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset);
+ if(!collections_summaries_op.ok()) {
+ res->set(collections_summaries_op.code(), collections_summaries_op.error());
+ return false;
+ }
+
+ nlohmann::json json_response = collections_summaries_op.get();
res->set_200(json_response.dump());
return true;
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 86482919..e67dd556 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -301,7 +301,7 @@ TEST_F(CollectionManagerTest, ParallelCollectionCreation) {
int64_t prev_id = INT32_MAX;
- for(auto coll: collectionManager.get_collections()) {
+ for(auto coll: collectionManager.get_collections().get()) {
// collections are sorted by ID, in descending order
ASSERT_TRUE(coll->get_collection_id() < prev_id);
prev_id = coll->get_collection_id();
@@ -355,7 +355,7 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {
}
TEST_F(CollectionManagerTest, GetAllCollections) {
- std::vector<Collection*> collection_vec = collectionManager.get_collections();
+ std::vector<Collection*> collection_vec = collectionManager.get_collections().get();
ASSERT_EQ(1, collection_vec.size());
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
@@ -369,7 +369,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {
})"_json;
collectionManager.create_collection(new_schema);
- collection_vec = collectionManager.get_collections();
+ collection_vec = collectionManager.get_collections().get();
ASSERT_EQ(2, collection_vec.size());
// most recently created collection first
@@ -1104,7 +1104,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
ASSERT_TRUE(drop_op.ok());
// try to list collections now
- nlohmann::json summaries = cmanager.get_collection_summaries();
+ nlohmann::json summaries = cmanager.get_collection_summaries().get();
ASSERT_EQ(0, summaries.size());
// remap alias to another non-existing collection
@@ -1172,7 +1172,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {
cmanager.create_collection("collection" + std::to_string(i), 4, schema, "points").get();
}
- ASSERT_EQ(100, cmanager.get_collections().size());
+ ASSERT_EQ(100, cmanager.get_collections().get().size());
cmanager.dispose();
delete new_store;
@@ -1181,7 +1181,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {
cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
- ASSERT_EQ(100, cmanager.get_collections().size());
+ ASSERT_EQ(100, cmanager.get_collections().get().size());
for(size_t i = 0; i < 100; i++) {
collectionManager.drop_collection("collection" + std::to_string(i));
@@ -1907,4 +1907,84 @@ TEST_F(CollectionManagerTest, CollectionCreationWithMetadata) {
expected_meta_json["created_at"] = actual_json["created_at"];
ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
+}
+
+TEST_F(CollectionManagerTest, CollectionPagination) {
+ //remove all collections first
+ auto collections = collectionManager.get_collections().get();
+ for(auto collection : collections) {
+ collectionManager.drop_collection(collection->get_name());
+ }
+
+ //create few collections
+ for(size_t i = 0; i < 5; i++) {
+ nlohmann::json coll_json = R"({
+ "name": "cp",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+ coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);
+ auto coll_op = collectionManager.create_collection(coll_json);
+ ASSERT_TRUE(coll_op.ok());
+ }
+
+ uint32_t limit = 0, offset = 0;
+
+ //limit collections by 2
+ limit=2;
+ auto collection_op = collectionManager.get_collections(limit);
+ auto collections_vec = collection_op.get();
+ ASSERT_EQ(2, collections_vec.size());
+ ASSERT_EQ("cp2", collections_vec[0]->get_name());
+ ASSERT_EQ("cp5", collections_vec[1]->get_name());
+
+ //get 2 collection from offset 3
+ offset=3;
+ collection_op = collectionManager.get_collections(limit, offset);
+ collections_vec = collection_op.get();
+ ASSERT_EQ(2, collections_vec.size());
+ ASSERT_EQ("cp1", collections_vec[0]->get_name());
+ ASSERT_EQ("cp4", collections_vec[1]->get_name());
+
+ //get all collection except first
+ offset=1; limit=0;
+ collection_op = collectionManager.get_collections(limit, offset);
+ collections_vec = collection_op.get();
+ ASSERT_EQ(4, collections_vec.size());
+ ASSERT_EQ("cp5", collections_vec[0]->get_name());
+ ASSERT_EQ("cp3", collections_vec[1]->get_name());
+ ASSERT_EQ("cp1", collections_vec[2]->get_name());
+ ASSERT_EQ("cp4", collections_vec[3]->get_name());
+
+ //get last collection
+ offset=4, limit=1;
+ collection_op = collectionManager.get_collections(limit, offset);
+ collections_vec = collection_op.get();
+ ASSERT_EQ(1, collections_vec.size());
+ ASSERT_EQ("cp4", collections_vec[0]->get_name());
+
+ //if limit is greater than number of collection then return all from offset
+ offset=0; limit=8;
+ collection_op = collectionManager.get_collections(limit, offset);
+ collections_vec = collection_op.get();
+ ASSERT_EQ(5, collections_vec.size());
+ ASSERT_EQ("cp2", collections_vec[0]->get_name());
+ ASSERT_EQ("cp5", collections_vec[1]->get_name());
+ ASSERT_EQ("cp3", collections_vec[2]->get_name());
+ ASSERT_EQ("cp1", collections_vec[3]->get_name());
+ ASSERT_EQ("cp4", collections_vec[4]->get_name());
+
+ offset=3; limit=4;
+ collection_op = collectionManager.get_collections(limit, offset);
+ collections_vec = collection_op.get();
+ ASSERT_EQ(2, collections_vec.size());
+ ASSERT_EQ("cp1", collections_vec[0]->get_name());
+ ASSERT_EQ("cp4", collections_vec[1]->get_name());
+
+ //invalid offset
+ offset=6; limit=0;
+ collection_op = collectionManager.get_collections(limit, offset);
+ ASSERT_FALSE(collection_op.ok());
+ ASSERT_EQ("Invalid offset param.", collection_op.error());
}
\ No newline at end of file
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 446f16b0..cfb725d9 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -1555,4 +1555,77 @@ TEST_F(CoreAPIUtilsTest, DeleteNonExistingDoc) {
req->params["ignore_not_found"] = "true";
del_remove_document(req, res);
ASSERT_EQ(200, res->status_code);
+}
+
+TEST_F(CoreAPIUtilsTest, CollectionsPagination) {
+ //remove all collections first
+ auto collections = collectionManager.get_collections().get();
+ for(auto collection : collections) {
+ collectionManager.drop_collection(collection->get_name());
+ }
+
+ //create few collections
+ for(size_t i = 0; i < 5; i++) {
+ nlohmann::json coll_json = R"({
+ "name": "cp",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+ coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);
+ auto coll_op = collectionManager.create_collection(coll_json);
+ ASSERT_TRUE(coll_op.ok());
+ }
+
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["offset"] = "0";
+ req->params["limit"] = "1";
+
+ nlohmann::json expected_meta_json = R"(
+ {
+ "created_at":1663234047,
+ "default_sorting_field":"",
+ "enable_nested_fields":false,
+ "fields":[
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"title",
+ "optional":false,
+ "sort":false,
+ "stem":false,
+ "type":"string"
+ }
+ ],
+ "name":"cp2",
+ "num_documents":0,
+ "symbols_to_index":[],
+ "token_separators":[]
+ }
+ )"_json;
+
+ get_collections(req, resp);
+
+ auto actual_json = nlohmann::json::parse(resp->body);
+ expected_meta_json["created_at"] = actual_json[0]["created_at"];
+
+ ASSERT_EQ(expected_meta_json.dump(), actual_json[0].dump());
+
+ //invalid offset string
+ req->params["offset"] = "0a";
+ get_collections(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body);
+
+ //invalid limit string
+ req->params["offset"] = "0";
+ req->params["limit"] = "-1";
+ get_collections(req, resp);
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);
}
\ No newline at end of file