Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support pagination via offset parameter.
Kishore Nallan5029d375ff11
diff --git a/include/collection.h b/include/collection.hindex 3b50c435..97e39e5d 100644--- a/include/collection.h+++ b/include/collection.h@@ -453,7 +453,8 @@ public:const uint64_t search_time_start_us = 0,const text_match_type_t match_type = max_score,const size_t facet_sample_percent = 100,- const size_t facet_sample_threshold = 0) const;+ const size_t facet_sample_threshold = 0,+ const size_t page_offset = UINT32_MAX) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex 2a18bc9f..6c4d66fe 100644--- a/include/index.h+++ b/include/index.h@@ -109,7 +109,7 @@ struct search_args {std::vector<uint32_t> num_typos;size_t max_facet_values;size_t per_page;- size_t page;+ size_t offset;token_ordering token_order;std::vector<bool> prefixes;size_t drop_tokens_threshold;@@ -164,7 +164,7 @@ struct search_args {search_fields(search_fields), match_type(match_type), filter_tree_root(filter_tree_root), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),- page(page), token_order(token_order), prefixes(prefixes),+ offset(offset), token_order(token_order), prefixes(prefixes),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),@@ -637,7 +637,7 @@ public:const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,const size_t per_page,- const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,+ const size_t offset, const token_ordering token_order, const std::vector<bool>& prefixes,const size_t drop_tokens_threshold, size_t& all_result_ids_len,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,std::vector<std::vector<art_leaf*>>& searched_queries,diff --git a/src/collection.cpp b/src/collection.cppindex b5310d46..7609e151 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1063,7 +1063,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const uint64_t search_time_start_us,const text_match_type_t match_type,const size_t facet_sample_percent,- const size_t facet_sample_threshold) const {+ const size_t facet_sample_threshold,+ const size_t page_offset) const {std::shared_lock lock(mutex);@@ -1326,7 +1327,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}// check for valid pagination- if(page < 1) {+ if(page != 0 && page < 1) {std::string message = "Page must be an integer of value greater than 0.";return Option<nlohmann::json>(422, message);}@@ -1336,7 +1337,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(422, message);}- if((page * per_page) > limit_hits) {+ size_t offset = (page != 0) ? (per_page * (page - 1)) : page_offset;+ size_t fetch_size = offset + per_page;++ if(fetch_size > limit_hits) {std::string message = "Only upto " + std::to_string(limit_hits) + " hits can be fetched. " +"Ensure that `page` and `per_page` parameters are within this range.";return Option<nlohmann::json>(422, message);@@ -1346,9 +1350,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// ensure that `max_hits` never exceeds number of documents in collectionif(search_fields.size() <= 1 || raw_query == "*") {- max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());+ max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());} else {- max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());+ max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());}if(token_order == NOT_SET) {@@ -1505,7 +1509,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,match_type,filter_tree_root, facets, included_ids, excluded_ids,sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,- per_page, page, token_order, prefixes,+ per_page, offset, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field,prioritize_exact_match, prioritize_token_position,@@ -1639,10 +1643,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,facet_query_last_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();}- const long start_result_index = (page - 1) * per_page;+ const long start_result_index = offset;// `end_result_index` could be -1 when max_hits is 0- const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;+ const long end_result_index = std::min(fetch_size, std::min(max_hits, result_group_kvs.size())) - 1;// handle which fields have to be highlighteddiff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0475dfbe..3231475a 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -675,6 +675,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *LIMIT_HITS = "limit_hits";const char *PER_PAGE = "per_page";const char *PAGE = "page";+ const char *OFFSET = "offset";+ const char *LIMIT = "limit";const char *RANK_TOKENS_BY = "rank_tokens_by";const char *INCLUDE_FIELDS = "include_fields";const char *EXCLUDE_FIELDS = "exclude_fields";@@ -756,7 +758,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::vector<std::string> facet_fields;std::vector<sort_by> sort_fields;size_t per_page = 10;- size_t page = 1;+ size_t page = 0;+ size_t offset = UINT32_MAX;token_ordering token_order = NOT_SET;std::string vector_query;@@ -809,7 +812,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{SNIPPET_THRESHOLD, &snippet_threshold},{HIGHLIGHT_AFFIX_NUM_TOKENS, &highlight_affix_num_tokens},{PAGE, &page},+ {OFFSET, &offset},{PER_PAGE, &per_page},+ {LIMIT, &per_page},{GROUP_LIMIT, &group_limit},{SEARCH_CUTOFF_MS, &search_cutoff_ms},{MAX_EXTRA_PREFIX, &max_extra_prefix},@@ -947,6 +952,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reper_page = 0;}+ if(req_params[PAGE].empty() && req_params[OFFSET].empty()) {+ page = 1;+ }+include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());exclude_fields.insert(exclude_fields_vec.begin(), exclude_fields_vec.end());@@ -1030,7 +1039,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restart_ts,match_type,facet_sample_percent,- facet_sample_threshold+ facet_sample_threshold,+ offset);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(@@ -1049,7 +1059,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reresult["search_time_ms"] = timeMillis;}- result["page"] = page;+ if(page != 0) {+ result["page"] = page;+ } else {+ result["offset"] = offset;+ }+results_json_str = result.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);//LOG(INFO) << "Time taken: " << timeMillis << "ms";diff --git a/src/index.cpp b/src/index.cppindex 90ae73a0..b72b6ca8 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2217,7 +2217,7 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->included_ids, search_params->excluded_ids,search_params->sort_fields_std, search_params->num_typos,search_params->topster, search_params->curated_topster,- search_params->per_page, search_params->page, search_params->token_order,+ search_params->per_page, search_params->offset, search_params->token_order,search_params->prefixes, search_params->drop_tokens_threshold,search_params->all_result_ids_len, search_params->groups_processed,search_params->searched_queries,@@ -2678,7 +2678,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,const size_t per_page,- const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,+ const size_t offset, const token_ordering token_order, const std::vector<bool>& prefixes,const size_t drop_tokens_threshold, size_t& all_result_ids_len,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,std::vector<std::vector<art_leaf*>>& searched_queries,@@ -2713,6 +2713,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consreturn Option(true);}+ size_t fetch_size = offset + per_page;+std::set<uint32_t> curated_ids;std::map<size_t, std::map<size_t, uint32_t>> included_ids_map; // outer pos => inner pos => list of IDsstd::vector<uint32_t> included_ids_vec;@@ -2793,7 +2795,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consgroups_processed[distinct_id]++;}- if (result_ids.size() == page * per_page) {+ if (result_ids.size() == fetch_size) {break;}@@ -2816,7 +2818,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscollate_included_ids({}, included_ids_map, curated_topster, searched_queries);if (!vector_query.field_name.empty()) {- auto k = std::max<size_t>(vector_query.k, per_page * page);+ auto k = std::max<size_t>(vector_query.k, fetch_size);if(vector_query.query_doc_given) {// since we will omit the query doc from resultsk++;@@ -3082,7 +3084,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consVectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);auto& field_vector_index = vector_index.at(vector_query.field_name);std::vector<std::pair<float, size_t>> dist_labels;- auto k = std::max<size_t>(vector_query.k, per_page * page);+ auto k = std::max<size_t>(vector_query.k, fetch_size);if(field_vector_index->distance_type == cosine) {std::vector<float> normalized_q(vector_query.values.size());diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 9978a071..1bf5a88b 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -617,6 +617,124 @@ TEST_F(CoreAPIUtilsTest, PresetSingleSearch) {collectionManager.drop_collection("coll1");}+TEST_F(CoreAPIUtilsTest, SearchPagination) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ for(size_t i = 0; i < 20; i++) {+ nlohmann::json doc;+ doc["name"] = "Title " + std::to_string(i);+ doc["points"] = i;+ coll1->add(doc.dump(), CREATE);+ }++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ nlohmann::json body;++ // without any pagination params, default is top 10 records by sort order++ body["searches"] = nlohmann::json::array();+ nlohmann::json search;+ search["collection"] = "coll1";+ search["q"] = "title";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);++ req->body = body.dump();+ nlohmann::json embedded_params;+ req->embedded_params_vec.push_back(embedded_params);++ post_multi_search(req, res);+ nlohmann::json results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(10, results["hits"].size());+ ASSERT_EQ(19, results["hits"][0]["document"]["points"].get<size_t>());+ ASSERT_EQ(1, results["page"].get<size_t>());++ // when offset is used we should expect the same but "offset" should be returned in response+ search.clear();+ req->params.clear();+ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "title";+ search["offset"] = "1";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);+ req->body = body.dump();++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(10, results["hits"].size());+ ASSERT_EQ(18, results["hits"][0]["document"]["points"].get<size_t>());+ ASSERT_EQ(1, results["offset"].get<size_t>());++ // use limit to restrict page size+ search.clear();+ req->params.clear();+ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "title";+ search["offset"] = "1";+ search["limit"] = "5";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);+ req->body = body.dump();++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(18, results["hits"][0]["document"]["points"].get<size_t>());+ ASSERT_EQ(1, results["offset"].get<size_t>());++ // when page is -1+ search.clear();+ req->params.clear();+ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "title";+ search["page"] = "-1";+ search["limit"] = "5";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);+ req->body = body.dump();++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(400, results["code"].get<size_t>());+ ASSERT_EQ("Parameter `page` must be an unsigned integer.", results["error"].get<std::string>());++ // when offset is -1+ search.clear();+ req->params.clear();+ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "title";+ search["offset"] = "-1";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);+ req->body = body.dump();++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(400, results["code"].get<size_t>());+ ASSERT_EQ("Parameter `offset` must be an unsigned integer.", results["error"].get<std::string>());+}+TEST_F(CoreAPIUtilsTest, ExportWithFilter) {Collection *coll1;std::vector<field> fields = {field("title", field_types::STRING, false),