Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Reuse KV pointers instead of copying as objects.
kishorenc8ffb9c5154e4
diff --git a/include/collection.h b/include/collection.hindex c1ebc55e..4807d091 100644--- a/include/collection.h+++ b/include/collection.h@@ -147,7 +147,7 @@ private:std::string get_seq_id_key(uint32_t seq_id);void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,- const KV &field_order_kv, const nlohmann::json &document,+ const KV* field_order_kv, const nlohmann::json &document,StringUtils & string_utils, size_t snippet_threshold,bool highlighted_fully,highlight_t &highlight);diff --git a/include/index.h b/include/index.hindex 9f2d2edd..994aa6e5 100644--- a/include/index.h+++ b/include/index.h@@ -32,17 +32,18 @@ struct search_args {facet_query_t facet_query;int num_typos;size_t max_facet_values;- size_t max_hits;size_t per_page;size_t page;token_ordering token_order;bool prefix;size_t drop_tokens_threshold;size_t typo_tokens_threshold;- std::vector<KV> raw_result_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;- std::vector<KV> override_result_kvs;+ Topster* topster;+ Topster* curated_topster;+ std::vector<KV*> raw_result_kvs;+ std::vector<KV*> override_result_kvs;Option<uint32_t> outcome;search_args(): outcome(0) {@@ -56,12 +57,20 @@ struct search_args {size_t drop_tokens_threshold, size_t typo_tokens_threshold):query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),- max_facet_values(max_facet_values), max_hits(max_hits), per_page(per_page),+ max_facet_values(max_facet_values), per_page(per_page),page(page), token_order(token_order), prefix(prefix),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),all_result_ids_len(0), outcome(0) {+ const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory+ topster = new Topster(topster_size);+ curated_topster = new Topster(topster_size);}++ ~search_args() {+ delete topster;+ delete curated_topster;+ };};struct index_record {@@ -155,7 +164,7 @@ private:const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,const int num_typos, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster & topster, uint32_t** all_result_ids,+ Topster* topster, uint32_t** all_result_ids,size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY,const bool prefix = false,const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,@@ -164,7 +173,7 @@ private:void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster & topster, uint32_t** all_result_ids,+ Topster* topster, uint32_t** all_result_ids,size_t & all_result_ids_len,const size_t typo_tokens_threshold);@@ -198,7 +207,7 @@ private:void collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,const std::vector<uint32_t> & included_ids,- Topster & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);+ Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);uint64_t facet_token_hash(const field & a_field, const std::string &token);@@ -220,10 +229,12 @@ public:facet_query_t & facet_query,const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,const std::vector<sort_by> & sort_fields_std, const int num_typos,- const size_t max_hits, const size_t per_page, const size_t page, const token_ordering token_order,- const bool prefix, const size_t drop_tokens_threshold, std::vector<KV> & raw_result_kvs,+ Topster* topster, Topster* curated_topster,+ const size_t per_page, const size_t page, const token_ordering token_order,+ const bool prefix, const size_t drop_tokens_threshold,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,- std::vector<KV> & override_result_kvs, const size_t typo_tokens_threshold);+ std::vector<KV*> & raw_result_kvs, std::vector<KV*> & override_result_kvs,+ const size_t typo_tokens_threshold);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);@@ -235,7 +246,7 @@ public:std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,- const uint32_t total_cost, Topster &topster, const std::vector<art_leaf *> & query_suggestion,+ const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,const uint32_t *result_ids, const size_t result_size) const;static int32_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);@@ -278,7 +289,7 @@ public:bool processed; // prevents spurious wake up of the main threadbool terminate; // used for interrupting the thread during tear down- search_args search_params;+ search_args* search_params;static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,const art_leaf *token_leaf, uint32_t doc_index);diff --git a/include/topster.h b/include/topster.hindex ab6d752b..bbc7f590 100644--- a/include/topster.h+++ b/include/topster.h@@ -257,11 +257,6 @@ struct Topster {std::tie(j->scores[0], j->scores[1], j->scores[2]);}- static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {- return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >- std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);- }-// topster must be sorted before iterated upon to remove dead array entriesvoid sort() {std::stable_sort(kvs, kvs+size, is_greater_kv);diff --git a/src/collection.cpp b/src/collection.cppindex 2e7b30f2..eee23762 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -608,15 +608,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const size_t max_hits = std::min((page * per_page), get_num_documents());std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results- std::vector<KV> raw_result_kvs;- std::vector<KV> override_result_kvs;+ std::vector<KV*> raw_result_kvs;+ std::vector<KV*> override_result_kvs;size_t total_found = 0;// send data to individual index threadssize_t index_id = 0;for(Index* index: indices) {- index->search_params = search_args(query, search_fields, filters, facets,+ index->search_params = new search_args(query, search_fields, filters, facets,index_to_included_ids[index_id], index_to_excluded_ids[index_id],sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,per_page, page, token_order, prefix,@@ -639,9 +639,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::index->cv.wait(lk, [index]{return index->processed;});}- if(!index->search_params.outcome.ok()) {- index_search_op = Option<nlohmann::json>(index->search_params.outcome.code(),- index->search_params.outcome.error());+ if(!index->search_params->outcome.ok()) {+ index_search_op = Option<nlohmann::json>(index->search_params->outcome.code(),+ index->search_params->outcome.error());}if(!index_search_op.ok()) {@@ -649,21 +649,21 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::continue;}- for(auto & field_order_kv: index->search_params.raw_result_kvs) {- field_order_kv.query_index += searched_queries.size();+ for(auto & field_order_kv: index->search_params->raw_result_kvs) {+ field_order_kv->query_index += searched_queries.size();raw_result_kvs.push_back(field_order_kv);}- for(auto & field_order_kv: index->search_params.override_result_kvs) {- field_order_kv.query_index += searched_queries.size();+ for(auto & field_order_kv: index->search_params->override_result_kvs) {+ field_order_kv->query_index += searched_queries.size();override_result_kvs.push_back(field_order_kv);}- searched_queries.insert(searched_queries.end(), index->search_params.searched_queries.begin(),- index->search_params.searched_queries.end());+ searched_queries.insert(searched_queries.end(), index->search_params->searched_queries.begin(),+ index->search_params->searched_queries.end());- for(size_t fi = 0; fi < index->search_params.facets.size(); fi++) {- auto & this_facet = index->search_params.facets[fi];+ for(size_t fi = 0; fi < index->search_params->facets.size(); fi++) {+ auto & this_facet = index->search_params->facets[fi];auto & acc_facet = facets[fi];for(auto & facet_kv: this_facet.result_map) {@@ -690,7 +690,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}- total_found += index->search_params.all_result_ids_len;+ total_found += index->search_params->all_result_ids_len;}if(!index_search_op.ok()) {@@ -698,13 +698,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}// All fields are sorted descending- std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_value);+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv);- // Sort based on position in overriden list+ // Sort based on position in overridden liststd::sort(override_result_kvs.begin(), override_result_kvs.end(),- [&id_pos_map](const KV & a, const KV & b) -> bool {- return id_pos_map[a.key] < id_pos_map[b.key];+ [&id_pos_map](const KV* a, const KV* b) -> bool {+ return id_pos_map[a->key] < id_pos_map[b->key];});@@ -713,15 +713,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::result["hits"] = nlohmann::json::array();result["found"] = total_found;- std::vector<KV> result_kvs;+ std::vector<KV*> result_kvs;size_t override_kv_index = 0;size_t raw_results_index = 0;// merge raw results and override resultswhile(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {if(override_kv_index < override_result_kvs.size() &&- id_pos_map.count(override_result_kvs[override_kv_index].key) != 0 &&- result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index].key]) {+ id_pos_map.count(override_result_kvs[override_kv_index]->key) != 0 &&+ result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {result_kvs.push_back(override_result_kvs[override_kv_index]);override_kv_index++;} else {@@ -746,7 +746,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// construct results arrayfor(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {const auto & field_order_kv = result_kvs[result_kvs_index];- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);nlohmann::json document;const Option<bool> & document_op = get_document_from_store(seq_id_key, document);@@ -818,8 +818,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::prune_document(document, include_fields, exclude_fields);wrapper_doc["document"] = document;- wrapper_doc["text_match"] = field_order_kv.match_score;- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;+ wrapper_doc["text_match"] = field_order_kv->match_score;+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;result["hits"].push_back(wrapper_doc);}@@ -918,6 +918,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::result["facet_counts"].push_back(facet_result);}+ // free search params+ for(Index* index: indices) {+ delete index->search_params;+ }+result["request_params"] = nlohmann::json::object();;result["request_params"]["per_page"] = per_page;result["request_params"]["q"] = query;@@ -966,7 +971,7 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_tvoid Collection::highlight_result(const field &search_field,const std::vector<std::vector<art_leaf *>> &searched_queries,- const KV & field_order_kv, const nlohmann::json & document,+ const KV* field_order_kv, const nlohmann::json & document,StringUtils & string_utils, size_t snippet_threshold,bool highlighted_fully,highlight_t & highlight) {@@ -974,15 +979,15 @@ void Collection::highlight_result(const field &search_field,spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;std::vector<art_leaf *> query_suggestion;- for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {+ for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.- Index* index = indices[field_order_kv.key % num_indices];+ Index* index = indices[field_order_kv->key % num_indices];art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);if(actual_leaf != nullptr) {query_suggestion.push_back(actual_leaf);std::vector<uint16_t> positions;- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv.key);+ uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);auto doc_indices = new uint32_t[1];doc_indices[0] = doc_index;leaf_to_indices.emplace(actual_leaf, doc_indices);@@ -1008,8 +1013,8 @@ void Collection::highlight_result(const field &search_field,continue;}- const Match & this_match = Match::match(field_order_kv.key, token_positions);- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);+ const Match & this_match = Match::match(field_order_kv->key, token_positions);+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);match_indices.emplace_back(this_match, this_match_score, array_index);}diff --git a/src/index.cpp b/src/index.cppindex 9ce1b396..1ff8f5bb 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -829,7 +829,7 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,- std::vector<std::vector<art_leaf*>> & searched_queries, Topster & topster,+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster* topster,uint32_t** all_result_ids, size_t & all_result_ids_len,const size_t typo_tokens_threshold) {const long long combination_limit = 10;@@ -1055,13 +1055,15 @@ void Index::run_search() {}// after the wait, we own the lock.- search(search_params.outcome, search_params.query, search_params.search_fields,- search_params.filters, search_params.facets, search_params.facet_query, search_params.included_ids,- search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,- search_params.max_hits, search_params.per_page, search_params.page, search_params.token_order,- search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,- search_params.all_result_ids_len, search_params.searched_queries, search_params.override_result_kvs,- search_params.typo_tokens_threshold);+ search(search_params->outcome, search_params->query, search_params->search_fields,+ search_params->filters, search_params->facets, search_params->facet_query, search_params->included_ids,+ search_params->excluded_ids, search_params->sort_fields_std, search_params->num_typos,+ search_params->topster, search_params->curated_topster,+ search_params->per_page, search_params->page, search_params->token_order,+ search_params->prefix, search_params->drop_tokens_threshold,+ search_params->all_result_ids_len, search_params->searched_queries,+ search_params->raw_result_kvs, search_params->override_result_kvs,+ search_params->typo_tokens_threshold);// hand control back to main threadprocessed = true;@@ -1075,7 +1077,7 @@ void Index::run_search() {void Index::collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,const std::vector<uint32_t> & included_ids,- Topster & curated_topster,+ Topster* curated_topster,std::vector<std::vector<art_leaf*>> & searched_queries) {if(included_ids.size() == 0) {@@ -1110,9 +1112,9 @@ void Index::collate_curated_ids(const std::string & query, const std::string & fleaf_to_indices.emplace(token_leaf, indices);}- // curated_topster.MAX_SIZE is initialized based on max_hits.+ // curated_topster->MAX_SIZE is initialized based on max_hits.// Even if override has more IDs, we should restrict to max hits.- size_t iter_size = std::min((size_t)curated_topster.MAX_SIZE, included_ids.size());+ size_t iter_size = std::min((size_t)curated_topster->MAX_SIZE, included_ids.size());for(size_t j=0; j<iter_size; j++) {const uint32_t seq_id = included_ids[j];@@ -1140,7 +1142,7 @@ void Index::collate_curated_ids(const std::string & query, const std::string & fscores[2] = int64_t(1);KV kv(field_id, searched_queries.size(), seq_id, seq_id, match_score, scores);- curated_topster.add(&kv);+ curated_topster->add(&kv);searched_queries.push_back(override_query);}@@ -1153,13 +1155,15 @@ void Index::search(Option<uint32_t> & outcome,std::vector<facet> & facets, facet_query_t & facet_query,const std::vector<uint32_t> & included_ids,const std::vector<uint32_t> & excluded_ids,- const std::vector<sort_by> & sort_fields_std, const int num_typos, const size_t max_hits,+ const std::vector<sort_by> & sort_fields_std, const int num_typos,+ Topster* topster,+ Topster* curated_topster,const size_t per_page, const size_t page, const token_ordering token_order,const bool prefix, const size_t drop_tokens_threshold,- std::vector<KV> & raw_result_kvs,size_t & all_result_ids_len,std::vector<std::vector<art_leaf*>> & searched_queries,- std::vector<KV> & override_result_kvs,+ std::vector<KV*> & raw_result_kvs,+ std::vector<KV*> & override_result_kvs,const size_t typo_tokens_threshold) {const size_t num_results = (page * per_page);@@ -1179,10 +1183,6 @@ void Index::search(Option<uint32_t> & outcome,//auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;- const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory- Topster topster(topster_size);- Topster curated_topster(topster_size);-if(query == "*") {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string & field = search_fields[0];@@ -1212,8 +1212,8 @@ void Index::search(Option<uint32_t> & outcome,do_facets(facets, facet_query, &included_ids[0], included_ids.size());// must be sorted before iterated upon to remove "empty" array entries- topster.sort();- curated_topster.sort();+ topster->sort();+ curated_topster->sort();std::set<uint32_t> ids_to_remove(included_ids.begin(), included_ids.end());ids_to_remove.insert(excluded_ids.begin(), excluded_ids.end());@@ -1221,26 +1221,26 @@ void Index::search(Option<uint32_t> & outcome,std::vector<uint32_t> dropped_ids;// loop through topster and remove elements from included and excluded id lists- for(uint32_t t = 0; t < topster.size && t < num_results; t++) {- KV* kv = topster.getKV(t);+ for(uint32_t t = 0; t < topster->size && t < num_results; t++) {+ KV* kv = topster->getKV(t);if(ids_to_remove.count(kv->key) != 0) {dropped_ids.push_back((uint32_t)kv->key);} else {- raw_result_kvs.push_back(*kv);+ raw_result_kvs.push_back(kv);}}- for(uint32_t t = 0; t < curated_topster.size && t < num_results; t++) {- KV* kv = curated_topster.getKV(t);- override_result_kvs.push_back(*kv);+ for(uint32_t t = 0; t < curated_topster->size && t < num_results; t++) {+ KV* kv = curated_topster->getKV(t);+ override_result_kvs.push_back(kv);}// for the ids that are dropped, remove their corresponding facet components from facet resultsdrop_facets(facets, dropped_ids);all_result_ids_len -= dropped_ids.size();- all_result_ids_len += curated_topster.size;+ all_result_ids_len += curated_topster->size;delete [] filter_ids;delete [] all_result_ids;@@ -1264,7 +1264,7 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, couint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,std::vector<std::vector<art_leaf*>> & searched_queries,- Topster & topster, uint32_t** all_result_ids, size_t & all_result_ids_len,+ Topster* topster, uint32_t** all_result_ids, size_t & all_result_ids_len,const token_ordering token_order, const bool prefix,const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) {std::vector<std::string> tokens;@@ -1438,7 +1438,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,- const uint8_t & field_id, const uint32_t total_cost, Topster & topster,+ const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const {@@ -1546,7 +1546,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}KV kv(field_id, query_index, seq_id, seq_id, match_score, scores);- topster.add(&kv);+ topster->add(&kv);}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();