Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Dynamic expansion of max hits

Jason Boscoc110f8ecfff1
diff --git a/include/collection.h b/include/collection.h
index c3ca6d85..66bade45 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -373,8 +373,7 @@ public:
const size_t group_limit = 0,
const std::string& highlight_start_tag="<mark>",
const std::string& highlight_end_tag="</mark>",
- std::vector<size_t> query_by_weights={},
- size_t max_hits=250);
+ std::vector<size_t> query_by_weights={});
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/src/collection.cpp b/src/collection.cpp
index faa0aa8f..b16a2649 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -512,8 +512,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const size_t group_limit,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
- std::vector<size_t> query_by_weights,
- size_t max_hits) {
+ std::vector<size_t> query_by_weights) {
if(query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
@@ -682,26 +681,27 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<sort_by> sort_fields_std;
- for(const sort_by & _sort_field: sort_fields) {
- if(_sort_field.name != sort_field_const::text_match && sort_schema.count(_sort_field.name) == 0) {
- std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";
+ for(const sort_by& _sort_field: sort_fields) {
+ sort_by sort_field_std(_sort_field.name, _sort_field.order);
+
+ if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {
+ std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";
return Option<nlohmann::json>(404, error);
}
- if(sort_schema.count(_sort_field.name) != 0 && sort_schema.at(_sort_field.name).optional) {
- std::string error = "Cannot sort by `" + _sort_field.name + "` as it is defined as an optional field.";
+ if(sort_schema.count(sort_field_std.name) != 0 && sort_schema.at(sort_field_std.name).optional) {
+ std::string error = "Cannot sort by `" + sort_field_std.name + "` as it is defined as an optional field.";
return Option<nlohmann::json>(400, error);
}
- std::string sort_order = _sort_field.order;
- StringUtils::toupper(sort_order);
+ StringUtils::toupper(sort_field_std.order);
- if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {
- std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";
+ if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {
+ std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";
return Option<nlohmann::json>(400, error);
}
- sort_fields_std.emplace_back(_sort_field.name, sort_order);
+ sort_fields_std.emplace_back(sort_field_std);
}
/*
@@ -743,11 +743,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return Option<nlohmann::json>(422, message);
}
+ size_t max_hits;
+
// ensure that `max_hits` never exceeds number of documents in collection
if(search_fields.size() <= 1 || query == "*") {
max_hits = std::min((page * per_page), get_num_documents());
} else {
- max_hits = std::min(max_hits, get_num_documents());
+ max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());
}
std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results
diff --git a/src/core_api.cpp b/src/core_api.cpp
index a12b466b..ae97670e 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -277,7 +277,6 @@ bool get_search(http_req & req, http_res & res) {
const char *GROUP_BY = "group_by";
const char *GROUP_LIMIT = "group_limit";
- const char *MAX_HITS = "max_hits";
const char *PER_PAGE = "per_page";
const char *PAGE = "page";
const char *CALLBACK = "callback";
@@ -329,13 +328,6 @@ bool get_search(http_req & req, http_res & res) {
req.params[FACET_QUERY] = "";
}
- if(req.params.count(MAX_HITS) == 0) {
- req.params[MAX_HITS] = "250";
- } else if(req.params[MAX_HITS] == "all") {
- // set a high value such that engine will default to size of collection
- req.params[MAX_HITS] = "100000000";
- }
-
if(req.params.count(SNIPPET_THRESHOLD) == 0) {
req.params[SNIPPET_THRESHOLD] = "30";
}
@@ -430,11 +422,6 @@ bool get_search(http_req & req, http_res & res) {
return false;
}
- if(!StringUtils::is_uint32_t(req.params[MAX_HITS])) {
- res.set_400("Parameter `" + std::string(MAX_HITS) + "` must be an unsigned integer.");
- return false;
- }
-
if(!StringUtils::is_uint32_t(req.params[MAX_FACET_VALUES])) {
res.set_400("Parameter `" + std::string(MAX_FACET_VALUES) + "` must be an unsigned integer.");
return false;
@@ -544,8 +531,7 @@ bool get_search(http_req & req, http_res & res) {
static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),
req.params[HIGHLIGHT_START_TAG],
req.params[HIGHLIGHT_END_TAG],
- query_by_weights,
- static_cast<size_t>(std::stol(req.params[MAX_HITS]))
+ query_by_weights
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index 8621c703..29bfd318 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -197,7 +197,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
art_tree *t = search_index.at(field_name);
- std::vector<std::string> strings = document[field_name];
+ const std::vector<std::string>& strings = document[field_name];
index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
auto num_tree = numerical_index.at(field_name);
@@ -1557,7 +1557,6 @@ void Index::search(Option<uint32_t> & outcome,
}
uint32_t doc_index = leaves[0]->values->ids.indexOf(seq_id);
-
if (doc_index == leaves[0]->values->ids.getLength()) {
continue;
}
@@ -1565,12 +1564,7 @@ void Index::search(Option<uint32_t> & outcome,
token_bits |= 1UL << token_index; // sets nth bit
//LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);
- uint32_t start_offset = leaves[0]->values->offset_index.at(doc_index);
- uint32_t end_offset = (doc_index == leaves[0]->values->ids.getLength() - 1) ?
- leaves[0]->values->offsets.getLength() :
- leaves[0]->values->offset_index.at(doc_index+1);
-
- words_present += (end_offset - start_offset);
+ words_present += 1;
/*if(!leaves.empty()) {
LOG(INFO) << "tok: " << leaves[0]->key;
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index ec170030..cb31e14e 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -552,7 +552,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10,
- pinned_hits, {}, {}, 0, "", "", {}, 350).get();
+ pinned_hits, {}, {}, 0, "", "", {}).get();
ASSERT_EQ(311, results["found"].get<size_t>());
ASSERT_EQ(11, results["hits"].size());