Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support pagination via offset parameter.

Kishore Nallan5029d375ff11
diff --git a/include/collection.h b/include/collection.h
index 3b50c435..97e39e5d 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -453,7 +453,8 @@ public:
const uint64_t search_time_start_us = 0,
const text_match_type_t match_type = max_score,
const size_t facet_sample_percent = 100,
- const size_t facet_sample_threshold = 0) const;
+ const size_t facet_sample_threshold = 0,
+ const size_t page_offset = UINT32_MAX) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --git a/include/index.h b/include/index.h
index 2a18bc9f..6c4d66fe 100644
--- a/include/index.h
+++ b/include/index.h
@@ -109,7 +109,7 @@ struct search_args {
std::vector<uint32_t> num_typos;
size_t max_facet_values;
size_t per_page;
- size_t page;
+ size_t offset;
token_ordering token_order;
std::vector<bool> prefixes;
size_t drop_tokens_threshold;
@@ -164,7 +164,7 @@ struct search_args {
search_fields(search_fields), match_type(match_type), filter_tree_root(filter_tree_root), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
- page(page), token_order(token_order), prefixes(prefixes),
+ offset(offset), token_order(token_order), prefixes(prefixes),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),
@@ -637,7 +637,7 @@ public:
const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,
const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,
const size_t per_page,
- const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,
+ const size_t offset, const token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
diff --git a/src/collection.cpp b/src/collection.cpp
index b5310d46..7609e151 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1063,7 +1063,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const uint64_t search_time_start_us,
const text_match_type_t match_type,
const size_t facet_sample_percent,
- const size_t facet_sample_threshold) const {
+ const size_t facet_sample_threshold,
+ const size_t page_offset) const {
std::shared_lock lock(mutex);
@@ -1326,7 +1327,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
// check for valid pagination
- if(page < 1) {
+ if(page != 0 && page < 1) {
std::string message = "Page must be an integer of value greater than 0.";
return Option<nlohmann::json>(422, message);
}
@@ -1336,7 +1337,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
return Option<nlohmann::json>(422, message);
}
- if((page * per_page) > limit_hits) {
+ size_t offset = (page != 0) ? (per_page * (page - 1)) : page_offset;
+ size_t fetch_size = offset + per_page;
+
+ if(fetch_size > limit_hits) {
std::string message = "Only upto " + std::to_string(limit_hits) + " hits can be fetched. " +
"Ensure that `page` and `per_page` parameters are within this range.";
return Option<nlohmann::json>(422, message);
@@ -1346,9 +1350,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
// ensure that `max_hits` never exceeds number of documents in collection
if(search_fields.size() <= 1 || raw_query == "*") {
- max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());
+ max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());
} else {
- max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());
+ max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());
}
if(token_order == NOT_SET) {
@@ -1505,7 +1509,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
match_type,
filter_tree_root, facets, included_ids, excluded_ids,
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
- per_page, page, token_order, prefixes,
+ per_page, offset, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
group_by_fields, group_limit, default_sorting_field,
prioritize_exact_match, prioritize_token_position,
@@ -1639,10 +1643,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
facet_query_last_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();
}
- const long start_result_index = (page - 1) * per_page;
+ const long start_result_index = offset;
// `end_result_index` could be -1 when max_hits is 0
- const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;
+ const long end_result_index = std::min(fetch_size, std::min(max_hits, result_group_kvs.size())) - 1;
// handle which fields have to be highlighted
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 0475dfbe..3231475a 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -675,6 +675,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *LIMIT_HITS = "limit_hits";
const char *PER_PAGE = "per_page";
const char *PAGE = "page";
+ const char *OFFSET = "offset";
+ const char *LIMIT = "limit";
const char *RANK_TOKENS_BY = "rank_tokens_by";
const char *INCLUDE_FIELDS = "include_fields";
const char *EXCLUDE_FIELDS = "exclude_fields";
@@ -756,7 +758,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::vector<std::string> facet_fields;
std::vector<sort_by> sort_fields;
size_t per_page = 10;
- size_t page = 1;
+ size_t page = 0;
+ size_t offset = UINT32_MAX;
token_ordering token_order = NOT_SET;
std::string vector_query;
@@ -809,7 +812,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{SNIPPET_THRESHOLD, &snippet_threshold},
{HIGHLIGHT_AFFIX_NUM_TOKENS, &highlight_affix_num_tokens},
{PAGE, &page},
+ {OFFSET, &offset},
{PER_PAGE, &per_page},
+ {LIMIT, &per_page},
{GROUP_LIMIT, &group_limit},
{SEARCH_CUTOFF_MS, &search_cutoff_ms},
{MAX_EXTRA_PREFIX, &max_extra_prefix},
@@ -947,6 +952,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
per_page = 0;
}
+ if(req_params[PAGE].empty() && req_params[OFFSET].empty()) {
+ page = 1;
+ }
+
include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());
exclude_fields.insert(exclude_fields_vec.begin(), exclude_fields_vec.end());
@@ -1030,7 +1039,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
start_ts,
match_type,
facet_sample_percent,
- facet_sample_threshold
+ facet_sample_threshold,
+ offset
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -1049,7 +1059,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
result["search_time_ms"] = timeMillis;
}
- result["page"] = page;
+ if(page != 0) {
+ result["page"] = page;
+ } else {
+ result["offset"] = offset;
+ }
+
results_json_str = result.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);
//LOG(INFO) << "Time taken: " << timeMillis << "ms";
diff --git a/src/index.cpp b/src/index.cpp
index 90ae73a0..b72b6ca8 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2217,7 +2217,7 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
search_params->included_ids, search_params->excluded_ids,
search_params->sort_fields_std, search_params->num_typos,
search_params->topster, search_params->curated_topster,
- search_params->per_page, search_params->page, search_params->token_order,
+ search_params->per_page, search_params->offset, search_params->token_order,
search_params->prefixes, search_params->drop_tokens_threshold,
search_params->all_result_ids_len, search_params->groups_processed,
search_params->searched_queries,
@@ -2678,7 +2678,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,
const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,
const size_t per_page,
- const size_t page, const token_ordering token_order, const std::vector<bool>& prefixes,
+ const size_t offset, const token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold, size_t& all_result_ids_len,
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries,
@@ -2713,6 +2713,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
return Option(true);
}
+ size_t fetch_size = offset + per_page;
+
std::set<uint32_t> curated_ids;
std::map<size_t, std::map<size_t, uint32_t>> included_ids_map; // outer pos => inner pos => list of IDs
std::vector<uint32_t> included_ids_vec;
@@ -2793,7 +2795,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
groups_processed[distinct_id]++;
}
- if (result_ids.size() == page * per_page) {
+ if (result_ids.size() == fetch_size) {
break;
}
@@ -2816,7 +2818,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
if (!vector_query.field_name.empty()) {
- auto k = std::max<size_t>(vector_query.k, per_page * page);
+ auto k = std::max<size_t>(vector_query.k, fetch_size);
if(vector_query.query_doc_given) {
// since we will omit the query doc from results
k++;
@@ -3082,7 +3084,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
- auto k = std::max<size_t>(vector_query.k, per_page * page);
+ auto k = std::max<size_t>(vector_query.k, fetch_size);
if(field_vector_index->distance_type == cosine) {
std::vector<float> normalized_q(vector_query.values.size());
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 9978a071..1bf5a88b 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -617,6 +617,124 @@ TEST_F(CoreAPIUtilsTest, PresetSingleSearch) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CoreAPIUtilsTest, SearchPagination) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string" },
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ for(size_t i = 0; i < 20; i++) {
+ nlohmann::json doc;
+ doc["name"] = "Title " + std::to_string(i);
+ doc["points"] = i;
+ coll1->add(doc.dump(), CREATE);
+ }
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ nlohmann::json body;
+
+ // without any pagination params, default is top 10 records by sort order
+
+ body["searches"] = nlohmann::json::array();
+ nlohmann::json search;
+ search["collection"] = "coll1";
+ search["q"] = "title";
+ search["query_by"] = "name";
+ search["sort_by"] = "points:desc";
+ body["searches"].push_back(search);
+
+ req->body = body.dump();
+ nlohmann::json embedded_params;
+ req->embedded_params_vec.push_back(embedded_params);
+
+ post_multi_search(req, res);
+ nlohmann::json results = nlohmann::json::parse(res->body)["results"][0];
+ ASSERT_EQ(10, results["hits"].size());
+ ASSERT_EQ(19, results["hits"][0]["document"]["points"].get<size_t>());
+ ASSERT_EQ(1, results["page"].get<size_t>());
+
+ // when offset is used we should expect the same but "offset" should be returned in response
+ search.clear();
+ req->params.clear();
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "title";
+ search["offset"] = "1";
+ search["query_by"] = "name";
+ search["sort_by"] = "points:desc";
+ body["searches"].push_back(search);
+ req->body = body.dump();
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+ ASSERT_EQ(10, results["hits"].size());
+ ASSERT_EQ(18, results["hits"][0]["document"]["points"].get<size_t>());
+ ASSERT_EQ(1, results["offset"].get<size_t>());
+
+ // use limit to restrict page size
+ search.clear();
+ req->params.clear();
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "title";
+ search["offset"] = "1";
+ search["limit"] = "5";
+ search["query_by"] = "name";
+ search["sort_by"] = "points:desc";
+ body["searches"].push_back(search);
+ req->body = body.dump();
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+ ASSERT_EQ(5, results["hits"].size());
+ ASSERT_EQ(18, results["hits"][0]["document"]["points"].get<size_t>());
+ ASSERT_EQ(1, results["offset"].get<size_t>());
+
+ // when page is -1
+ search.clear();
+ req->params.clear();
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "title";
+ search["page"] = "-1";
+ search["limit"] = "5";
+ search["query_by"] = "name";
+ search["sort_by"] = "points:desc";
+ body["searches"].push_back(search);
+ req->body = body.dump();
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+ ASSERT_EQ(400, results["code"].get<size_t>());
+ ASSERT_EQ("Parameter `page` must be an unsigned integer.", results["error"].get<std::string>());
+
+ // when offset is -1
+ search.clear();
+ req->params.clear();
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "title";
+ search["offset"] = "-1";
+ search["query_by"] = "name";
+ search["sort_by"] = "points:desc";
+ body["searches"].push_back(search);
+ req->body = body.dump();
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+ ASSERT_EQ(400, results["code"].get<size_t>());
+ ASSERT_EQ("Parameter `offset` must be an unsigned integer.", results["error"].get<std::string>());
+}
+
TEST_F(CoreAPIUtilsTest, ExportWithFilter) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),