Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use `max_hits` parameter for consistent pagination.
kishorenc59f95dbf67d8
diff --git a/include/collection.h b/include/collection.hindex 66bade45..c3ca6d85 100644--- a/include/collection.h+++ b/include/collection.h@@ -373,7 +373,8 @@ public:const size_t group_limit = 0,const std::string& highlight_start_tag="<mark>",const std::string& highlight_end_tag="</mark>",- std::vector<size_t> query_by_weights={});+ std::vector<size_t> query_by_weights={},+ size_t max_hits=250);Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/src/collection.cpp b/src/collection.cppindex 2eda9305..5e6060d1 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -511,7 +511,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const size_t group_limit,const std::string& highlight_start_tag,const std::string& highlight_end_tag,- std::vector<size_t> query_by_weights) {+ std::vector<size_t> query_by_weights,+ size_t max_hits) {if(query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");@@ -741,8 +742,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return Option<nlohmann::json>(422, message);}- // ensure that (page * per_page) never exceeds number of documents in collection- const size_t max_hits = std::min((page * per_page), get_num_documents());+ // ensure that `max_hits` never exceeds number of documents in collection+ if(search_fields.size() <= 1 || query == "*") {+ max_hits = std::min((page * per_page), get_num_documents());+ } else {+ max_hits = std::min(max_hits, get_num_documents());+ }std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the resultsstd::vector<std::vector<KV*>> raw_result_kvs;@@ -916,7 +921,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}const long start_result_index = (page - 1) * per_page;- const long end_result_index = std::min(max_hits, result_group_kvs.size()) - 1; // could be -1 when max_hits is 0++ // `end_result_index` could be -1 when max_hits is 0+ const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;nlohmann::json result = nlohmann::json::object();diff --git a/src/core_api.cpp b/src/core_api.cppindex 315ad166..63ec3d76 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -269,6 +269,7 @@ bool get_search(http_req & req, http_res & res) {const char *GROUP_BY = "group_by";const char *GROUP_LIMIT = "group_limit";+ const char *MAX_HITS = "max_hits";const char *PER_PAGE = "per_page";const char *PAGE = "page";const char *CALLBACK = "callback";@@ -320,6 +321,13 @@ bool get_search(http_req & req, http_res & res) {req.params[FACET_QUERY] = "";}+ if(req.params.count(MAX_HITS) == 0) {+ req.params[MAX_HITS] = "250";+ } else if(req.params[MAX_HITS] == "all") {+ // set a high value such that engine will default to size of collection+ req.params[MAX_HITS] = "100000000";+ }+if(req.params.count(SNIPPET_THRESHOLD) == 0) {req.params[SNIPPET_THRESHOLD] = "30";}@@ -414,6 +422,11 @@ bool get_search(http_req & req, http_res & res) {return false;}+ if(!StringUtils::is_uint32_t(req.params[MAX_HITS])) {+ res.set_400("Parameter `" + std::string(MAX_HITS) + "` must be an unsigned integer.");+ return false;+ }+if(!StringUtils::is_uint32_t(req.params[MAX_FACET_VALUES])) {res.set_400("Parameter `" + std::string(MAX_FACET_VALUES) + "` must be an unsigned integer.");return false;@@ -523,7 +536,8 @@ bool get_search(http_req & req, http_res & res) {static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),req.params[HIGHLIGHT_START_TAG],req.params[HIGHLIGHT_END_TAG],- query_by_weights+ query_by_weights,+ static_cast<size_t>(std::stol(req.params[MAX_HITS])));uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex add62207..6390b371 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -639,10 +639,15 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());// facet query longer than a token is correctly matched with typo tolerance- results = coll1->search("phone", {"categories"}, "", facets, sort_fields, 0, 10, 1,+ // also ensure that setting per_page = 0 works fine++ results = coll1->search("phone", {"categories"}, "", facets, sort_fields, 0, 0, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "categories:cellx").get();+ ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<uint32_t>());+ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex c9af87fe..ec170030 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -552,7 +552,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- pinned_hits, {}).get();+ pinned_hits, {}, {}, 0, "", "", {}, 350).get();ASSERT_EQ(311, results["found"].get<size_t>());ASSERT_EQ(11, results["hits"].size());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex eae963d1..ff6040d0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -944,6 +944,78 @@ TEST_F(CollectionTest, MultipleFields) {collectionManager.drop_collection("coll_mul_fields");}+TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {+ Collection *coll_mul_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("starring_facet", field_types::STRING, true),+ field("cast", field_types::STRING_ARRAY, false),+ field("points", field_types::INT32, false)+ };++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_mul_fields->add(json_line);+ }++ infile.close();++ query_fields = {"title", "starring"};+ std::vector<std::string> facets;++ spp::sparse_hash_set<std::string> empty;+ nlohmann::json results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 1,+ FREQUENCY, true, 1000, empty, empty, 10).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(7, results["found"].get<int>());++ // cannot fetch more than in-built limit of 250+ auto res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 251, 1,+ FREQUENCY, true, 1000, empty, empty, 10);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ(422, res_op.code());+ ASSERT_STREQ("Only upto 250 hits can be fetched per page.", res_op.error().c_str());++ // when page number is not valid+ res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 10, 0,+ FREQUENCY, true, 1000, empty, empty, 10);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ(422, res_op.code());+ ASSERT_STREQ("Page must be an integer of value greater than 0.", res_op.error().c_str());++ // do pagination++ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 1,+ FREQUENCY, true, 1000, empty, empty, 10).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(7, results["found"].get<int>());++ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 2,+ FREQUENCY, true, 1000, empty, empty, 10).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(7, results["found"].get<int>());++ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 3,+ FREQUENCY, true, 1000, empty, empty, 10).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(7, results["found"].get<int>());++ collectionManager.drop_collection("coll_mul_fields");+}+std::vector<nlohmann::json> import_res_to_json(const std::vector<std::string>& imported_results) {std::vector<nlohmann::json> out;@@ -2445,7 +2517,7 @@ TEST_F(CollectionTest, OptionalFields) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionTest, ReturnsResultsBasedOnPerPageParam) {+TEST_F(CollectionTest, WildcardQueryReturnsResultsBasedOnPerPageParam) {std::vector<std::string> facets;spp::sparse_hash_set<std::string> empty;nlohmann::json results = collection->search("*", query_fields, "", facets, sort_fields, 0, 12, 1,