Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Dynamic expansion of max hits
Jason Boscoc110f8ecfff1
diff --git a/include/collection.h b/include/collection.hindex c3ca6d85..66bade45 100644--- a/include/collection.h+++ b/include/collection.h@@ -373,8 +373,7 @@ public:const size_t group_limit = 0,const std::string& highlight_start_tag="<mark>",const std::string& highlight_end_tag="</mark>",- std::vector<size_t> query_by_weights={},- size_t max_hits=250);+ std::vector<size_t> query_by_weights={});Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/src/collection.cpp b/src/collection.cppindex faa0aa8f..b16a2649 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -512,8 +512,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const size_t group_limit,const std::string& highlight_start_tag,const std::string& highlight_end_tag,- std::vector<size_t> query_by_weights,- size_t max_hits) {+ std::vector<size_t> query_by_weights) {if(query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");@@ -682,26 +681,27 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<sort_by> sort_fields_std;- for(const sort_by & _sort_field: sort_fields) {- if(_sort_field.name != sort_field_const::text_match && sort_schema.count(_sort_field.name) == 0) {- std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";+ for(const sort_by& _sort_field: sort_fields) {+ sort_by sort_field_std(_sort_field.name, _sort_field.order);++ if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {+ std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";return Option<nlohmann::json>(404, error);}- if(sort_schema.count(_sort_field.name) != 0 && sort_schema.at(_sort_field.name).optional) {- std::string error = "Cannot sort by `" + _sort_field.name + "` as it is defined as an optional field.";+ if(sort_schema.count(sort_field_std.name) != 0 && sort_schema.at(sort_field_std.name).optional) {+ std::string error = "Cannot sort by `" + sort_field_std.name + "` as it is defined as an optional field.";return Option<nlohmann::json>(400, error);}- std::string sort_order = _sort_field.order;- StringUtils::toupper(sort_order);+ StringUtils::toupper(sort_field_std.order);- if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {- std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";+ if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {+ std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";return Option<nlohmann::json>(400, error);}- sort_fields_std.emplace_back(_sort_field.name, sort_order);+ sort_fields_std.emplace_back(sort_field_std);}/*@@ -743,11 +743,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return Option<nlohmann::json>(422, message);}+ size_t max_hits;+// ensure that `max_hits` never exceeds number of documents in collectionif(search_fields.size() <= 1 || query == "*") {max_hits = std::min((page * per_page), get_num_documents());} else {- max_hits = std::min(max_hits, get_num_documents());+ max_hits = std::min(std::max((page * per_page), max_hits), get_num_documents());}std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the resultsdiff --git a/src/core_api.cpp b/src/core_api.cppindex a12b466b..ae97670e 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -277,7 +277,6 @@ bool get_search(http_req & req, http_res & res) {const char *GROUP_BY = "group_by";const char *GROUP_LIMIT = "group_limit";- const char *MAX_HITS = "max_hits";const char *PER_PAGE = "per_page";const char *PAGE = "page";const char *CALLBACK = "callback";@@ -329,13 +328,6 @@ bool get_search(http_req & req, http_res & res) {req.params[FACET_QUERY] = "";}- if(req.params.count(MAX_HITS) == 0) {- req.params[MAX_HITS] = "250";- } else if(req.params[MAX_HITS] == "all") {- // set a high value such that engine will default to size of collection- req.params[MAX_HITS] = "100000000";- }-if(req.params.count(SNIPPET_THRESHOLD) == 0) {req.params[SNIPPET_THRESHOLD] = "30";}@@ -430,11 +422,6 @@ bool get_search(http_req & req, http_res & res) {return false;}- if(!StringUtils::is_uint32_t(req.params[MAX_HITS])) {- res.set_400("Parameter `" + std::string(MAX_HITS) + "` must be an unsigned integer.");- return false;- }-if(!StringUtils::is_uint32_t(req.params[MAX_FACET_VALUES])) {res.set_400("Parameter `" + std::string(MAX_FACET_VALUES) + "` must be an unsigned integer.");return false;@@ -544,8 +531,7 @@ bool get_search(http_req & req, http_res & res) {static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),req.params[HIGHLIGHT_START_TAG],req.params[HIGHLIGHT_END_TAG],- query_by_weights,- static_cast<size_t>(std::stol(req.params[MAX_HITS]))+ query_by_weights);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 8621c703..29bfd318 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -197,7 +197,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tnum_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {art_tree *t = search_index.at(field_name);- std::vector<std::string> strings = document[field_name];+ const std::vector<std::string>& strings = document[field_name];index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);} else if(field_pair.second.type == field_types::INT32_ARRAY) {auto num_tree = numerical_index.at(field_name);@@ -1557,7 +1557,6 @@ void Index::search(Option<uint32_t> & outcome,}uint32_t doc_index = leaves[0]->values->ids.indexOf(seq_id);-if (doc_index == leaves[0]->values->ids.getLength()) {continue;}@@ -1565,12 +1564,7 @@ void Index::search(Option<uint32_t> & outcome,token_bits |= 1UL << token_index; // sets nth bit//LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);- uint32_t start_offset = leaves[0]->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == leaves[0]->values->ids.getLength() - 1) ?- leaves[0]->values->offsets.getLength() :- leaves[0]->values->offset_index.at(doc_index+1);-- words_present += (end_offset - start_offset);+ words_present += 1;/*if(!leaves.empty()) {LOG(INFO) << "tok: " << leaves[0]->key;diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex ec170030..cb31e14e 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -552,7 +552,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10,- pinned_hits, {}, {}, 0, "", "", {}, 350).get();+ pinned_hits, {}, {}, 0, "", "", {}).get();ASSERT_EQ(311, results["found"].get<size_t>());ASSERT_EQ(11, results["hits"].size());