Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use `max_hits` parameter for consistent pagination.

kishorenc59f95dbf67d8
diff --git a/include/collection.h b/include/collection.h
index 66bade45..c3ca6d85 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -373,7 +373,8 @@ public:
const size_t group_limit = 0,
const std::string& highlight_start_tag="<mark>",
const std::string& highlight_end_tag="</mark>",
- std::vector<size_t> query_by_weights={});
+ std::vector<size_t> query_by_weights={},
+ size_t max_hits=250);
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/src/collection.cpp b/src/collection.cpp
index 2eda9305..5e6060d1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -511,7 +511,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const size_t group_limit,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
- std::vector<size_t> query_by_weights) {
+ std::vector<size_t> query_by_weights,
+ size_t max_hits) {
if(query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
@@ -741,8 +742,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return Option<nlohmann::json>(422, message);
}
- // ensure that (page * per_page) never exceeds number of documents in collection
- const size_t max_hits = std::min((page * per_page), get_num_documents());
+ // ensure that `max_hits` never exceeds number of documents in collection
+ if(search_fields.size() <= 1 || query == "*") {
+ max_hits = std::min((page * per_page), get_num_documents());
+ } else {
+ max_hits = std::min(max_hits, get_num_documents());
+ }
std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results
std::vector<std::vector<KV*>> raw_result_kvs;
@@ -916,7 +921,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
const long start_result_index = (page - 1) * per_page;
- const long end_result_index = std::min(max_hits, result_group_kvs.size()) - 1; // could be -1 when max_hits is 0
+
+ // `end_result_index` could be -1 when max_hits is 0
+ const long end_result_index = std::min((page * per_page), std::min(max_hits, result_group_kvs.size())) - 1;
nlohmann::json result = nlohmann::json::object();
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 315ad166..63ec3d76 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -269,6 +269,7 @@ bool get_search(http_req & req, http_res & res) {
const char *GROUP_BY = "group_by";
const char *GROUP_LIMIT = "group_limit";
+ const char *MAX_HITS = "max_hits";
const char *PER_PAGE = "per_page";
const char *PAGE = "page";
const char *CALLBACK = "callback";
@@ -320,6 +321,13 @@ bool get_search(http_req & req, http_res & res) {
req.params[FACET_QUERY] = "";
}
+ if(req.params.count(MAX_HITS) == 0) {
+ req.params[MAX_HITS] = "250";
+ } else if(req.params[MAX_HITS] == "all") {
+ // set a high value such that engine will default to size of collection
+ req.params[MAX_HITS] = "100000000";
+ }
+
if(req.params.count(SNIPPET_THRESHOLD) == 0) {
req.params[SNIPPET_THRESHOLD] = "30";
}
@@ -414,6 +422,11 @@ bool get_search(http_req & req, http_res & res) {
return false;
}
+ if(!StringUtils::is_uint32_t(req.params[MAX_HITS])) {
+ res.set_400("Parameter `" + std::string(MAX_HITS) + "` must be an unsigned integer.");
+ return false;
+ }
+
if(!StringUtils::is_uint32_t(req.params[MAX_FACET_VALUES])) {
res.set_400("Parameter `" + std::string(MAX_FACET_VALUES) + "` must be an unsigned integer.");
return false;
@@ -523,7 +536,8 @@ bool get_search(http_req & req, http_res & res) {
static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),
req.params[HIGHLIGHT_START_TAG],
req.params[HIGHLIGHT_END_TAG],
- query_by_weights
+ query_by_weights,
+ static_cast<size_t>(std::stol(req.params[MAX_HITS]))
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index add62207..6390b371 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -639,10 +639,15 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
// facet query longer than a token is correctly matched with typo tolerance
- results = coll1->search("phone", {"categories"}, "", facets, sort_fields, 0, 10, 1,
+ // also ensure that setting per_page = 0 works fine
+
+ results = coll1->search("phone", {"categories"}, "", facets, sort_fields, 0, 0, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "categories:cellx").get();
+ ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<uint32_t>());
+
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index c9af87fe..ec170030 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -552,7 +552,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
- pinned_hits, {}).get();
+ pinned_hits, {}, {}, 0, "", "", {}, 350).get();
ASSERT_EQ(311, results["found"].get<size_t>());
ASSERT_EQ(11, results["hits"].size());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index eae963d1..ff6040d0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -944,6 +944,78 @@ TEST_F(CollectionTest, MultipleFields) {
collectionManager.drop_collection("coll_mul_fields");
}
+TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {
+ Collection *coll_mul_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("starring_facet", field_types::STRING, true),
+ field("cast", field_types::STRING_ARRAY, false),
+ field("points", field_types::INT32, false)
+ };
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+
+ query_fields = {"title", "starring"};
+ std::vector<std::string> facets;
+
+ spp::sparse_hash_set<std::string> empty;
+ nlohmann::json results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 1,
+ FREQUENCY, true, 1000, empty, empty, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(7, results["found"].get<int>());
+
+ // cannot fetch more than in-built limit of 250
+ auto res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 251, 1,
+ FREQUENCY, true, 1000, empty, empty, 10);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ(422, res_op.code());
+ ASSERT_STREQ("Only upto 250 hits can be fetched per page.", res_op.error().c_str());
+
+ // when page number is not valid
+ res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 10, 0,
+ FREQUENCY, true, 1000, empty, empty, 10);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ(422, res_op.code());
+ ASSERT_STREQ("Page must be an integer of value greater than 0.", res_op.error().c_str());
+
+ // do pagination
+
+ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 1,
+ FREQUENCY, true, 1000, empty, empty, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(7, results["found"].get<int>());
+
+ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 2,
+ FREQUENCY, true, 1000, empty, empty, 10).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(7, results["found"].get<int>());
+
+ results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, 0, 3, 3,
+ FREQUENCY, true, 1000, empty, empty, 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(7, results["found"].get<int>());
+
+ collectionManager.drop_collection("coll_mul_fields");
+}
+
std::vector<nlohmann::json> import_res_to_json(const std::vector<std::string>& imported_results) {
std::vector<nlohmann::json> out;
@@ -2445,7 +2517,7 @@ TEST_F(CollectionTest, OptionalFields) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionTest, ReturnsResultsBasedOnPerPageParam) {
+TEST_F(CollectionTest, WildcardQueryReturnsResultsBasedOnPerPageParam) {
std::vector<std::string> facets;
spp::sparse_hash_set<std::string> empty;
nlohmann::json results = collection->search("*", query_fields, "", facets, sort_fields, 0, 12, 1,