Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support upto 500 search results.

Kishore Nallan3d38b63d7f2a
diff --git a/include/collection.h b/include/collection.h
index 4f1a2e56..879eab8c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -102,7 +102,7 @@ public:
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
static const int MAX_SEARCH_TOKENS = 10;
- static const int MAX_RESULTS = 100;
+ static const int MAX_RESULTS = 500;
// strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
enum {SNIPPET_STR_ABOVE_LEN = 30};
diff --git a/include/index.h b/include/index.h
index bbe8a564..101b9aee 100644
--- a/include/index.h
+++ b/include/index.h
@@ -30,7 +30,7 @@ struct search_args {
size_t page;
token_ordering token_order;
bool prefix;
- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
+ std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -87,13 +87,13 @@ private:
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
const int num_typos, const size_t num_results,
std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster<100> & topster, uint32_t** all_result_ids,
+ Topster<512> & topster, uint32_t** all_result_ids,
size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);
void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster<100> & topster, size_t & total_results, uint32_t** all_result_ids,
+ Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,
size_t & all_result_ids_len, const size_t & max_results, const bool prefix);
void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
@@ -132,19 +132,19 @@ public:
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix,
- std::vector<std::pair<int, Topster<100>::KV>> & field_order_kv,
+ std::vector<std::pair<int, Topster<512>::KV>> & field_order_kv,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint32_t total_cost,
- Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion,
+ Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,
const uint32_t *result_ids, const size_t result_size) const;
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);
static const int MAX_SEARCH_TOKENS = 10;
- static const int MAX_RESULTS = 100;
+ static const int SEARCH_LIMIT_NUM = 100; // for limiting number of results on multiple candidates / query rewrites
// strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
enum {SNIPPET_STR_ABOVE_LEN = 30};
diff --git a/include/topster.h b/include/topster.h
index 347d1ca4..8b51ab14 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -11,7 +11,7 @@
/*
* Remembers the max-K elements seen so far using a min-heap
*/
-template <size_t MAX_SIZE=100>
+template <size_t MAX_SIZE=512>
struct Topster {
struct KV {
uint16_t query_index;
diff --git a/src/collection.cpp b/src/collection.cpp
index ba680906..62751a18 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -303,7 +303,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// all search queries that were used for generating the results
std::vector<std::vector<art_leaf*>> searched_queries;
- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
+ std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;
size_t total_found = 0;
// send data to individual index threads
@@ -357,7 +357,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// All fields are sorted descending
std::sort(field_order_kvs.begin(), field_order_kvs.end(),
- [](const std::pair<int, Topster<100>::KV> & a, const std::pair<int, Topster<100>::KV> & b) {
+ [](const std::pair<int, Topster<512>::KV> & a, const std::pair<int, Topster<512>::KV> & b) {
return std::tie(a.second.match_score, a.second.primary_attr, a.second.secondary_attr, a.first, a.second.key) >
std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);
});
diff --git a/src/index.cpp b/src/index.cpp
index c76c7241..1a8ab447 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -279,7 +279,7 @@ void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t
void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_by> & sort_fields,
std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,
- std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,
size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
const size_t & max_results, const bool prefix) {
const size_t combination_limit = 10;
@@ -599,7 +599,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
const std::string & simple_filter_query, std::vector<facet> & facets,
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page, const token_ordering token_order,
- const bool prefix, std::vector<std::pair<int, Topster<100>::KV>> & field_order_kvs,
+ const bool prefix, std::vector<std::pair<int, Topster<512>::KV>> & field_order_kvs,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {
const size_t num_results = (page * per_page);
@@ -619,7 +619,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
uint32_t* all_result_ids = nullptr;
for(int i = 0; i < search_fields.size(); i++) {
- Topster<100> topster;
+ Topster<512> topster;
const std::string & field = search_fields[i];
// proceed to query search only when no filters are provided or when filtering produces results
if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
@@ -655,13 +655,13 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
void Index::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ Topster<512> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
const token_ordering token_order, const bool prefix) {
std::vector<std::string> tokens;
StringUtils::split(query, tokens, " ");
const int max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- const size_t max_results = Index::MAX_RESULTS;
+ const size_t max_results = Index::SEARCH_LIMIT_NUM;
size_t total_results = topster.size;
@@ -819,7 +819,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index,
- const uint32_t total_cost, Topster<100> & topster,
+ const uint32_t total_cost, Topster<512> & topster,
const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const {