Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support upto 500 search results.
Kishore Nallan3d38b63d7f2a
diff --git a/include/collection.h b/include/collection.hindex 4f1a2e56..879eab8c 100644--- a/include/collection.h+++ b/include/collection.h@@ -102,7 +102,7 @@ public:Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);static const int MAX_SEARCH_TOKENS = 10;- static const int MAX_RESULTS = 100;+ static const int MAX_RESULTS = 500;// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionenum {SNIPPET_STR_ABOVE_LEN = 30};diff --git a/include/index.h b/include/index.hindex bbe8a564..101b9aee 100644--- a/include/index.h+++ b/include/index.h@@ -30,7 +30,7 @@ struct search_args {size_t page;token_ordering token_order;bool prefix;- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;+ std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;@@ -87,13 +87,13 @@ private:std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,const int num_typos, const size_t num_results,std::vector<std::vector<art_leaf*>> & searched_queries,- Topster<100> & topster, uint32_t** all_result_ids,+ Topster<512> & topster, uint32_t** all_result_ids,size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster<100> & topster, size_t & total_results, uint32_t** all_result_ids,+ Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,size_t & all_result_ids_len, const size_t & max_results, const bool prefix);void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,@@ -132,19 +132,19 @@ public:std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page,const token_ordering token_order, const bool prefix,- std::vector<std::pair<int, Topster<100>::KV>> & field_order_kv,+ std::vector<std::pair<int, Topster<512>::KV>> & field_order_kv,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint32_t total_cost,- Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion,+ Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,const uint32_t *result_ids, const size_t result_size) const;Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);static const int MAX_SEARCH_TOKENS = 10;- static const int MAX_RESULTS = 100;+ static const int SEARCH_LIMIT_NUM = 100; // for limiting number of results on multiple candidates / query rewrites// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionenum {SNIPPET_STR_ABOVE_LEN = 30};diff --git a/include/topster.h b/include/topster.hindex 347d1ca4..8b51ab14 100644--- a/include/topster.h+++ b/include/topster.h@@ -11,7 +11,7 @@/** Remembers the max-K elements seen so far using a min-heap*/-template <size_t MAX_SIZE=100>+template <size_t MAX_SIZE=512>struct Topster {struct KV {uint16_t query_index;diff --git a/src/collection.cpp b/src/collection.cppindex ba680906..62751a18 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -303,7 +303,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// all search queries that were used for generating the resultsstd::vector<std::vector<art_leaf*>> searched_queries;- std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;+ std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;size_t total_found = 0;// send data to individual index threads@@ -357,7 +357,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// All fields are sorted descendingstd::sort(field_order_kvs.begin(), field_order_kvs.end(),- [](const std::pair<int, Topster<100>::KV> & a, const std::pair<int, Topster<100>::KV> & b) {+ [](const std::pair<int, Topster<512>::KV> & a, const std::pair<int, Topster<512>::KV> & b) {return std::tie(a.second.match_score, a.second.primary_attr, a.second.secondary_attr, a.first, a.second.key) >std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);});diff --git a/src/index.cpp b/src/index.cppindex c76c7241..1a8ab447 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -279,7 +279,7 @@ void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_tvoid Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,- std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,const size_t & max_results, const bool prefix) {const size_t combination_limit = 10;@@ -599,7 +599,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> sconst std::string & simple_filter_query, std::vector<facet> & facets,std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page, const token_ordering token_order,- const bool prefix, std::vector<std::pair<int, Topster<100>::KV>> & field_order_kvs,+ const bool prefix, std::vector<std::pair<int, Topster<512>::KV>> & field_order_kvs,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {const size_t num_results = (page * per_page);@@ -619,7 +619,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> suint32_t* all_result_ids = nullptr;for(int i = 0; i < search_fields.size(); i++) {- Topster<100> topster;+ Topster<512> topster;const std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces resultsif(simple_filter_query.size() == 0 || filter_ids_length > 0) {@@ -655,13 +655,13 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> svoid Index::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,+ Topster<512> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,const token_ordering token_order, const bool prefix) {std::vector<std::string> tokens;StringUtils::split(query, tokens, " ");const int max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;- const size_t max_results = Index::MAX_RESULTS;+ const size_t max_results = Index::SEARCH_LIMIT_NUM;size_t total_results = topster.size;@@ -819,7 +819,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index,- const uint32_t total_cost, Topster<100> & topster,+ const uint32_t total_cost, Topster<512> & topster,const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const {