Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use heap allocation rather than stack allocation for topster data.
Kishore Nallan4f1736124bd1
diff --git a/TODO.md b/TODO.mdindex ba6b6966..87cd8c58 100644--- a/TODO.md+++ b/TODO.md@@ -96,6 +96,7 @@- ~~gzip compress responses~~- ~~Have a LOG(ERROR) level~~- ~~Handle SIGTERM which is sent when process is killed~~+- Use snappy compression for storage- Exact search- NOT operator support- Log operationsdiff --git a/include/index.h b/include/index.hindex f5351647..26cc9f42 100644--- a/include/index.h+++ b/include/index.h@@ -33,7 +33,7 @@ struct search_args {token_ordering token_order;bool prefix;size_t drop_tokens_threshold;- std::vector<Topster<512>::KV*> field_order_kvs;+ std::vector<Topster<512>::KV> field_order_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;Option<uint32_t> outcome;@@ -142,7 +142,7 @@ public:std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page,const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,- std::vector<Topster<512>::KV*> & field_order_kv,+ std::vector<Topster<512>::KV> & field_order_kv,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);diff --git a/include/string_utils.h b/include/string_utils.hindex 0dfcfc9d..5672469a 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -9,14 +9,14 @@struct StringUtils {UErrorCode status;- icu::Transliterator* transliterator;+ //icu::Transliterator* transliterator;- StringUtils():status(U_ZERO_ERROR),- transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status)) {+ StringUtils(): status(U_ZERO_ERROR) {+ // transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))}~StringUtils() {- delete transliterator;+ //delete transliterator;}// Adapted from: http://stackoverflow.com/a/236180/131050diff --git a/include/topster.h b/include/topster.hindex 7d3e87c3..2f803c82 100644--- a/include/topster.h+++ b/include/topster.h@@ -21,8 +21,9 @@ struct Topster {uint64_t match_score;number_t primary_attr;number_t secondary_attr;- } data[MAX_SIZE];+ };+ KV *data;uint32_t size;spp::sparse_hash_map<uint64_t, KV*> keys;@@ -30,12 +31,23 @@ struct Topster {KV *kvs[MAX_SIZE];Topster(): size(0){+ data = new KV[MAX_SIZE];+for(size_t i=0; i<MAX_SIZE; i++) {+ data[i].field_id = 0;+ data[i].query_index = 0;+ data[i].key = 0;+ data[i].match_score = 0;+ data[i].primary_attr = number_t();+ data[i].secondary_attr = number_t();kvs[i] = &data[i];- kvs[i]->array_index = i;}}+ ~Topster() {+ delete [] data;+ }+static inline void swapMe(KV** a, KV** b) {KV *temp = *a;*a = *b;@@ -166,6 +178,11 @@ struct Topster {std::tie(j->match_score, j->primary_attr, j->secondary_attr, j->key);}+ static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {+ return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.key) >+ std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.key);+ }+// topster must be sorted before iterated upon to remove dead array entriesvoid sort() {std::stable_sort(std::begin(kvs), std::begin(kvs) + size, is_greater_kv);diff --git a/src/collection.cpp b/src/collection.cppindex 8f665f86..855860be 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -428,7 +428,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// all search queries that were used for generating the resultsstd::vector<std::vector<art_leaf*>> searched_queries;- std::vector<Topster<512>::KV*> field_order_kvs;+ std::vector<Topster<512>::KV> field_order_kvs;size_t total_found = 0;// send data to individual index threads@@ -463,8 +463,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- for(auto* field_order_kv: index->search_params.field_order_kvs) {- field_order_kv->query_index += searched_queries.size();+ for(auto & field_order_kv: index->search_params.field_order_kvs) {+ field_order_kv.query_index += searched_queries.size();field_order_kvs.push_back(field_order_kv);}@@ -497,7 +497,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}// All fields are sorted descending- std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv);+ std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv_value);nlohmann::json result = nlohmann::json::object();@@ -515,8 +515,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// construct results arrayfor(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {- const auto* field_order_kv = field_order_kvs[field_order_kv_index];- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);+ const auto & field_order_kv = field_order_kvs[field_order_kv_index];+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);std::string json_doc_str;StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);@@ -536,19 +536,19 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}wrapper_doc["document"] = document;- //wrapper_doc["match_score"] = field_order_kv->match_score;- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;+ //wrapper_doc["match_score"] = field_order_kv.match_score;+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;// highlight query words in the result- const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv->field_id];+ const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];field search_field = search_schema.at(field_name);if(search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;- for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {+ for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {std::vector<uint16_t> positions;- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv->key);+ uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);uint32_t *indices = new uint32_t[1];indices[0] = doc_index;leaf_to_indices.emplace(token_leaf, indices);@@ -556,7 +556,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// positions in the field of each token in the querystd::vector<std::vector<std::vector<uint16_t>>> array_token_positions;- Index::populate_token_positions(searched_queries[field_order_kv->query_index],+ Index::populate_token_positions(searched_queries[field_order_kv.query_index],leaf_to_indices, 0, array_token_positions);Match match;@@ -570,8 +570,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- const Match & this_match = Match::match(field_order_kv->key, token_positions);- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);+ const Match & this_match = Match::match(field_order_kv.key, token_positions);+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);if(this_match_score > match_score) {match_score = this_match_score;match = this_match;diff --git a/src/index.cpp b/src/index.cppindex 83ebd74e..c1bff470 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -600,7 +600,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vecstd::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page, const token_ordering token_order,const bool prefix, const size_t drop_tokens_threshold,- std::vector<Topster<512>::KV*> & field_order_kvs,+ std::vector<Topster<512>::KV> & field_order_kvs,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {const size_t num_results = (page * per_page);@@ -640,7 +640,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec// order of fields specified matter: matching docs from earlier fields are more importantfor(uint32_t t = 0; t < topster.size && t < num_results; t++) {Topster<512>::KV* kv = topster.getKV(t);- field_order_kvs.push_back(kv);+ field_order_kvs.push_back(*kv);}delete [] filter_ids;