Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix bug with offset/pagination affecting vector search.
Kishore Nalland569cbd75e31
diff --git a/include/collection.h b/include/collection.hindex 2f79bc68..65d51c0c 100644--- a/include/collection.h+++ b/include/collection.h@@ -463,7 +463,7 @@ public:const text_match_type_t match_type = max_score,const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,- const size_t page_offset = UINT32_MAX,+ const size_t page_offset = 0,const size_t vector_query_hits = 250) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex 5c1b77e0..2d87fa45 100644--- a/include/index.h+++ b/include/index.h@@ -150,7 +150,7 @@ struct search_args {filter_node_t* filter_tree_root, std::vector<facet>& facets,std::vector<std::pair<uint32_t, uint32_t>>& included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by>& sort_fields_std, facet_query_t facet_query, const std::vector<uint32_t>& num_typos,- size_t max_facet_values, size_t max_hits, size_t per_page, size_t page, token_ordering token_order,+ size_t max_facet_values, size_t max_hits, size_t per_page, size_t offset, token_ordering token_order,const std::vector<bool>& prefixes, size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit,const string& default_sorting_field, bool prioritize_exact_match,diff --git a/src/collection.cpp b/src/collection.cppindex c55bee7d..05bfb824 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1392,7 +1392,17 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(422, message);}- size_t offset = (page != 0) ? (per_page * (page - 1)) : page_offset;+ size_t offset = 0;++ if(page == 0 && page_offset != 0) {+ // if only offset is set, use that+ offset = page_offset;+ } else {+ // if both are set or none set, use page value (default is 1)+ size_t actual_page = (page == 0) ? 1 : page;+ offset = (per_page * (actual_page - 1));+ }+size_t fetch_size = offset + per_page;if(fetch_size > limit_hits) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 5323e1b0..f8d32b4e 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -783,7 +783,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::vector<sort_by> sort_fields;size_t per_page = 10;size_t page = 0;- size_t offset = UINT32_MAX;+ size_t offset = 0;token_ordering token_order = NOT_SET;std::string vector_query;@@ -978,14 +978,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reper_page = 0;}- if(!req_params[PAGE].empty() && page == 0 && offset == UINT32_MAX) {- return Option<bool>(422, "Parameter `page` must be an integer of value greater than 0.");- }-- if(req_params[PAGE].empty() && req_params[OFFSET].empty()) {- page = 1;- }-include_fields.insert(include_fields_vec.begin(), include_fields_vec.end());exclude_fields.insert(exclude_fields_vec.begin(), exclude_fields_vec.end());@@ -1097,10 +1089,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reresult["search_time_ms"] = timeMillis;}- if(page != 0) {- result["page"] = page;- } else {+ if(page == 0 && offset != 0) {result["offset"] = offset;+ } else {+ result["page"] = (page == 0) ? 1 : page;}results_json_str = result.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex e49e89f6..13d1bdab 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1054,11 +1054,11 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {ASSERT_EQ(422, res_op.code());ASSERT_STREQ("Only upto 250 hits can be fetched per page.", res_op.error().c_str());- // when page number is not valid- res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 10, 0,- FREQUENCY, {true}, 1000, empty, empty, 10);- ASSERT_FALSE(res_op.ok());- ASSERT_EQ(422, res_op.code());+ // when page number is zero, use the first page+ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 0,+ FREQUENCY, {true}, 1000, empty, empty, 10).get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(6, results["found"].get<int>());// do pagination@@ -3026,11 +3026,11 @@ TEST_F(CollectionTest, WildcardQueryReturnsResultsBasedOnPerPageParam) {ASSERT_EQ(422, res_op.code());ASSERT_STREQ("Only upto 250 hits can be fetched per page.", res_op.error().c_str());- // when page number is not valid- res_op = collection->search("*", query_fields, "", facets, sort_fields, {0}, 10, 0,- FREQUENCY, {false}, 1000, empty, empty, 10);- ASSERT_FALSE(res_op.ok());- ASSERT_EQ(422, res_op.code());+ // when page number is 0, just fetch first page+ results = collection->search("*", query_fields, "", facets, sort_fields, {0}, 10, 0,+ FREQUENCY, {false}, 1000, empty, empty, 10).get();+ ASSERT_EQ(10, results["hits"].size());+ ASSERT_EQ(25, results["found"].get<int>());// do paginationdiff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 95af7546..d24b44bf 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -768,7 +768,7 @@ TEST_F(CoreAPIUtilsTest, SearchPagination) {ASSERT_EQ(400, results["code"].get<size_t>());ASSERT_EQ("Parameter `offset` must be an unsigned integer.", results["error"].get<std::string>());- // when page is 0 and no offset is sent+ // when page is 0 and offset is NOT sent, we will treat as page=1search.clear();req->params.clear();body["searches"] = nlohmann::json::array();@@ -782,8 +782,29 @@ TEST_F(CoreAPIUtilsTest, SearchPagination) {post_multi_search(req, res);results = nlohmann::json::parse(res->body)["results"][0];- ASSERT_EQ(422, results["code"].get<size_t>());- ASSERT_EQ("Parameter `page` must be an integer of value greater than 0.", results["error"].get<std::string>());+ ASSERT_EQ(10, results["hits"].size());+ ASSERT_EQ(1, results["page"].get<size_t>());+ ASSERT_EQ(0, results.count("offset"));++ // when both page and offset are sent, use page+ search.clear();+ req->params.clear();+ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "title";+ search["page"] = "2";+ search["offset"] = "30";+ search["query_by"] = "name";+ search["sort_by"] = "points:desc";+ body["searches"].push_back(search);+ req->body = body.dump();++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];+ ASSERT_EQ(10, results["hits"].size());+ ASSERT_EQ(2, results["page"].get<size_t>());+ ASSERT_EQ(0, results.count("offset"));+}TEST_F(CoreAPIUtilsTest, ExportWithFilter) {