Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
pagination for collections GET (#1581)
Krunal Gandhi482858d05dc1
diff --git a/include/collection_manager.h b/include/collection_manager.hindex a88e3f7b..3473daa2 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -136,7 +136,7 @@ public:void add_to_collections(Collection* collection);- std::vector<Collection*> get_collections() const;+ Option<std::vector<Collection*>> get_collections(uint32_t limit = 0, uint32_t offset = 0) const;std::vector<std::string> get_collection_names() const;@@ -176,7 +176,7 @@ public:locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;- nlohmann::json get_collection_summaries() const;+ Option<nlohmann::json> get_collection_summaries(uint32_t limit = 0 , uint32_t offset = 0) const;Option<nlohmann::json> drop_collection(const std::string& collection_name,const bool remove_from_store = true,diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 7f1a8aa2..464dc1ea 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -572,20 +572,40 @@ locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uinreturn locked_resource_view_t<Collection>(mutex, nullptr);}-std::vector<Collection*> CollectionManager::get_collections() const {+Option<std::vector<Collection*>> CollectionManager::get_collections(uint32_t limit, uint32_t offset) const {std::shared_lock lock(mutex);std::vector<Collection*> collection_vec;- for(const auto& kv: collections) {- collection_vec.push_back(kv.second);+ auto collections_it = collections.begin();++ if(offset > 0) {+ if(offset >= collections.size()) {+ return Option<std::vector<Collection*>>(400, "Invalid offset param.");+ }++ std::advance(collections_it, offset);}- std::sort(std::begin(collection_vec), std::end(collection_vec),- [] (Collection* lhs, Collection* rhs) {- return lhs->get_collection_id() > rhs->get_collection_id();- });+ auto collections_end = collections.end();- return collection_vec;+ if(limit > 0 && (offset + limit < collections.size())) {+ collections_end = collections_it;+ std::advance(collections_end, limit);+ }++ for (collections_it; collections_it != collections_end; ++collections_it) {+ collection_vec.push_back(collections_it->second);+ }+++ if(offset == 0 && limit == 0) { //dont sort for paginated requests+ std::sort(std::begin(collection_vec), std::end(collection_vec),+ [](Collection *lhs, Collection *rhs) {+ return lhs->get_collection_id() > rhs->get_collection_id();+ });+ }++ return Option<std::vector<Collection*>>(collection_vec);}std::vector<std::string> CollectionManager::get_collection_names() const {@@ -1872,10 +1892,16 @@ ThreadPool* CollectionManager::get_thread_pool() const {return thread_pool;}-nlohmann::json CollectionManager::get_collection_summaries() const {+Option<nlohmann::json> CollectionManager::get_collection_summaries(uint32_t limit, uint32_t offset) const {std::shared_lock lock(mutex);- std::vector<Collection*> colls = get_collections();+ auto collections_op = get_collections(limit, offset);+ if(!collections_op.ok()) {+ return Option<nlohmann::json>(collections_op.code(), collections_op.error());+ }++ std::vector<Collection*> colls = collections_op.get();+nlohmann::json json_summaries = nlohmann::json::array();for(Collection* collection: colls) {@@ -1883,7 +1909,7 @@ nlohmann::json CollectionManager::get_collection_summaries() const {json_summaries.push_back(collection_json);}- return json_summaries;+ return Option<nlohmann::json>(json_summaries);}Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_json) {diff --git a/src/core_api.cpp b/src/core_api.cppindex 60926709..d54ccf60 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -181,7 +181,33 @@ index_operation_t get_index_operation(const std::string& action) {bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {CollectionManager & collectionManager = CollectionManager::get_instance();- nlohmann::json json_response = collectionManager.get_collection_summaries();++ uint32_t offset = 0, limit = 0;+ if(req->params.count("offset") != 0) {+ const auto &offset_str = req->params["offset"];+ if(!StringUtils::is_uint32_t(offset_str)) {+ res->set(400, "Offset param should be unsigned integer.");+ return false;+ }+ offset = std::stoi(offset_str);+ }++ if(req->params.count("limit") != 0) {+ const auto &limit_str = req->params["limit"];+ if(!StringUtils::is_uint32_t(limit_str)) {+ res->set(400, "Limit param should be unsigned integer.");+ return false;+ }+ limit = std::stoi(limit_str);+ }++ auto collections_summaries_op = collectionManager.get_collection_summaries(limit, offset);+ if(!collections_summaries_op.ok()) {+ res->set(collections_summaries_op.code(), collections_summaries_op.error());+ return false;+ }++ nlohmann::json json_response = collections_summaries_op.get();res->set_200(json_response.dump());return true;}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 86482919..e67dd556 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -301,7 +301,7 @@ TEST_F(CollectionManagerTest, ParallelCollectionCreation) {int64_t prev_id = INT32_MAX;- for(auto coll: collectionManager.get_collections()) {+ for(auto coll: collectionManager.get_collections().get()) {// collections are sorted by ID, in descending orderASSERT_TRUE(coll->get_collection_id() < prev_id);prev_id = coll->get_collection_id();@@ -355,7 +355,7 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {}TEST_F(CollectionManagerTest, GetAllCollections) {- std::vector<Collection*> collection_vec = collectionManager.get_collections();+ std::vector<Collection*> collection_vec = collectionManager.get_collections().get();ASSERT_EQ(1, collection_vec.size());ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());@@ -369,7 +369,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {})"_json;collectionManager.create_collection(new_schema);- collection_vec = collectionManager.get_collections();+ collection_vec = collectionManager.get_collections().get();ASSERT_EQ(2, collection_vec.size());// most recently created collection first@@ -1104,7 +1104,7 @@ TEST_F(CollectionManagerTest, Symlinking) {ASSERT_TRUE(drop_op.ok());// try to list collections now- nlohmann::json summaries = cmanager.get_collection_summaries();+ nlohmann::json summaries = cmanager.get_collection_summaries().get();ASSERT_EQ(0, summaries.size());// remap alias to another non-existing collection@@ -1172,7 +1172,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {cmanager.create_collection("collection" + std::to_string(i), 4, schema, "points").get();}- ASSERT_EQ(100, cmanager.get_collections().size());+ ASSERT_EQ(100, cmanager.get_collections().get().size());cmanager.dispose();delete new_store;@@ -1181,7 +1181,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {cmanager.init(new_store, 1.0, "auth_key", quit);cmanager.load(8, 1000);- ASSERT_EQ(100, cmanager.get_collections().size());+ ASSERT_EQ(100, cmanager.get_collections().get().size());for(size_t i = 0; i < 100; i++) {collectionManager.drop_collection("collection" + std::to_string(i));@@ -1907,4 +1907,84 @@ TEST_F(CollectionManagerTest, CollectionCreationWithMetadata) {expected_meta_json["created_at"] = actual_json["created_at"];ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());+}++TEST_F(CollectionManagerTest, CollectionPagination) {+ //remove all collections first+ auto collections = collectionManager.get_collections().get();+ for(auto collection : collections) {+ collectionManager.drop_collection(collection->get_name());+ }++ //create few collections+ for(size_t i = 0; i < 5; i++) {+ nlohmann::json coll_json = R"({+ "name": "cp",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;+ coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);+ auto coll_op = collectionManager.create_collection(coll_json);+ ASSERT_TRUE(coll_op.ok());+ }++ uint32_t limit = 0, offset = 0;++ //limit collections by 2+ limit=2;+ auto collection_op = collectionManager.get_collections(limit);+ auto collections_vec = collection_op.get();+ ASSERT_EQ(2, collections_vec.size());+ ASSERT_EQ("cp2", collections_vec[0]->get_name());+ ASSERT_EQ("cp5", collections_vec[1]->get_name());++ //get 2 collection from offset 3+ offset=3;+ collection_op = collectionManager.get_collections(limit, offset);+ collections_vec = collection_op.get();+ ASSERT_EQ(2, collections_vec.size());+ ASSERT_EQ("cp1", collections_vec[0]->get_name());+ ASSERT_EQ("cp4", collections_vec[1]->get_name());++ //get all collection except first+ offset=1; limit=0;+ collection_op = collectionManager.get_collections(limit, offset);+ collections_vec = collection_op.get();+ ASSERT_EQ(4, collections_vec.size());+ ASSERT_EQ("cp5", collections_vec[0]->get_name());+ ASSERT_EQ("cp3", collections_vec[1]->get_name());+ ASSERT_EQ("cp1", collections_vec[2]->get_name());+ ASSERT_EQ("cp4", collections_vec[3]->get_name());++ //get last collection+ offset=4, limit=1;+ collection_op = collectionManager.get_collections(limit, offset);+ collections_vec = collection_op.get();+ ASSERT_EQ(1, collections_vec.size());+ ASSERT_EQ("cp4", collections_vec[0]->get_name());++ //if limit is greater than number of collection then return all from offset+ offset=0; limit=8;+ collection_op = collectionManager.get_collections(limit, offset);+ collections_vec = collection_op.get();+ ASSERT_EQ(5, collections_vec.size());+ ASSERT_EQ("cp2", collections_vec[0]->get_name());+ ASSERT_EQ("cp5", collections_vec[1]->get_name());+ ASSERT_EQ("cp3", collections_vec[2]->get_name());+ ASSERT_EQ("cp1", collections_vec[3]->get_name());+ ASSERT_EQ("cp4", collections_vec[4]->get_name());++ offset=3; limit=4;+ collection_op = collectionManager.get_collections(limit, offset);+ collections_vec = collection_op.get();+ ASSERT_EQ(2, collections_vec.size());+ ASSERT_EQ("cp1", collections_vec[0]->get_name());+ ASSERT_EQ("cp4", collections_vec[1]->get_name());++ //invalid offset+ offset=6; limit=0;+ collection_op = collectionManager.get_collections(limit, offset);+ ASSERT_FALSE(collection_op.ok());+ ASSERT_EQ("Invalid offset param.", collection_op.error());}\ No newline at end of filediff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 446f16b0..cfb725d9 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1555,4 +1555,77 @@ TEST_F(CoreAPIUtilsTest, DeleteNonExistingDoc) {req->params["ignore_not_found"] = "true";del_remove_document(req, res);ASSERT_EQ(200, res->status_code);+}++TEST_F(CoreAPIUtilsTest, CollectionsPagination) {+ //remove all collections first+ auto collections = collectionManager.get_collections().get();+ for(auto collection : collections) {+ collectionManager.drop_collection(collection->get_name());+ }++ //create few collections+ for(size_t i = 0; i < 5; i++) {+ nlohmann::json coll_json = R"({+ "name": "cp",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;+ coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);+ auto coll_op = collectionManager.create_collection(coll_json);+ ASSERT_TRUE(coll_op.ok());+ }++ auto req = std::make_shared<http_req>();+ auto resp = std::make_shared<http_res>(nullptr);++ req->params["offset"] = "0";+ req->params["limit"] = "1";++ nlohmann::json expected_meta_json = R"(+ {+ "created_at":1663234047,+ "default_sorting_field":"",+ "enable_nested_fields":false,+ "fields":[+ {+ "facet":false,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"title",+ "optional":false,+ "sort":false,+ "stem":false,+ "type":"string"+ }+ ],+ "name":"cp2",+ "num_documents":0,+ "symbols_to_index":[],+ "token_separators":[]+ }+ )"_json;++ get_collections(req, resp);++ auto actual_json = nlohmann::json::parse(resp->body);+ expected_meta_json["created_at"] = actual_json[0]["created_at"];++ ASSERT_EQ(expected_meta_json.dump(), actual_json[0].dump());++ //invalid offset string+ req->params["offset"] = "0a";+ get_collections(req, resp);++ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body);++ //invalid limit string+ req->params["offset"] = "0";+ req->params["limit"] = "-1";+ get_collections(req, resp);+ ASSERT_EQ(400, resp->status_code);+ ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);}\ No newline at end of file