Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Reuse KV pointers instead of copying as objects.

kishorenc8ffb9c5154e4
diff --git a/include/collection.h b/include/collection.h
index c1ebc55e..4807d091 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -147,7 +147,7 @@ private:
std::string get_seq_id_key(uint32_t seq_id);
void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
- const KV &field_order_kv, const nlohmann::json &document,
+ const KV* field_order_kv, const nlohmann::json &document,
StringUtils & string_utils, size_t snippet_threshold,
bool highlighted_fully,
highlight_t &highlight);
diff --git a/include/index.h b/include/index.h
index 9f2d2edd..994aa6e5 100644
--- a/include/index.h
+++ b/include/index.h
@@ -32,17 +32,18 @@ struct search_args {
facet_query_t facet_query;
int num_typos;
size_t max_facet_values;
- size_t max_hits;
size_t per_page;
size_t page;
token_ordering token_order;
bool prefix;
size_t drop_tokens_threshold;
size_t typo_tokens_threshold;
- std::vector<KV> raw_result_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
- std::vector<KV> override_result_kvs;
+ Topster* topster;
+ Topster* curated_topster;
+ std::vector<KV*> raw_result_kvs;
+ std::vector<KV*> override_result_kvs;
Option<uint32_t> outcome;
search_args(): outcome(0) {
@@ -56,12 +57,20 @@ struct search_args {
size_t drop_tokens_threshold, size_t typo_tokens_threshold):
query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
- max_facet_values(max_facet_values), max_hits(max_hits), per_page(per_page),
+ max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefix(prefix),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
all_result_ids_len(0), outcome(0) {
+ const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
+ topster = new Topster(topster_size);
+ curated_topster = new Topster(topster_size);
}
+
+ ~search_args() {
+ delete topster;
+ delete curated_topster;
+ };
};
struct index_record {
@@ -155,7 +164,7 @@ private:
const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
const int num_typos, std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster & topster, uint32_t** all_result_ids,
+ Topster* topster, uint32_t** all_result_ids,
size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY,
const bool prefix = false,
const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
@@ -164,7 +173,7 @@ private:
void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,
const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster & topster, uint32_t** all_result_ids,
+ Topster* topster, uint32_t** all_result_ids,
size_t & all_result_ids_len,
const size_t typo_tokens_threshold);
@@ -198,7 +207,7 @@ private:
void collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,
const std::vector<uint32_t> & included_ids,
- Topster & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
+ Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
uint64_t facet_token_hash(const field & a_field, const std::string &token);
@@ -220,10 +229,12 @@ public:
facet_query_t & facet_query,
const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,
const std::vector<sort_by> & sort_fields_std, const int num_typos,
- const size_t max_hits, const size_t per_page, const size_t page, const token_ordering token_order,
- const bool prefix, const size_t drop_tokens_threshold, std::vector<KV> & raw_result_kvs,
+ Topster* topster, Topster* curated_topster,
+ const size_t per_page, const size_t page, const token_ordering token_order,
+ const bool prefix, const size_t drop_tokens_threshold,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,
- std::vector<KV> & override_result_kvs, const size_t typo_tokens_threshold);
+ std::vector<KV*> & raw_result_kvs, std::vector<KV*> & override_result_kvs,
+ const size_t typo_tokens_threshold);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
@@ -235,7 +246,7 @@ public:
std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);
void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
- const uint32_t total_cost, Topster &topster, const std::vector<art_leaf *> & query_suggestion,
+ const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,
const uint32_t *result_ids, const size_t result_size) const;
static int32_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
@@ -278,7 +289,7 @@ public:
bool processed; // prevents spurious wake up of the main thread
bool terminate; // used for interrupting the thread during tear down
- search_args search_params;
+ search_args* search_params;
static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,
const art_leaf *token_leaf, uint32_t doc_index);
diff --git a/include/topster.h b/include/topster.h
index ab6d752b..bbc7f590 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -257,11 +257,6 @@ struct Topster {
std::tie(j->scores[0], j->scores[1], j->scores[2]);
}
- static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {
- return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >
- std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);
- }
-
// topster must be sorted before iterated upon to remove dead array entries
void sort() {
std::stable_sort(kvs, kvs+size, is_greater_kv);
diff --git a/src/collection.cpp b/src/collection.cpp
index 2e7b30f2..eee23762 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -608,15 +608,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const size_t max_hits = std::min((page * per_page), get_num_documents());
std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results
- std::vector<KV> raw_result_kvs;
- std::vector<KV> override_result_kvs;
+ std::vector<KV*> raw_result_kvs;
+ std::vector<KV*> override_result_kvs;
size_t total_found = 0;
// send data to individual index threads
size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = search_args(query, search_fields, filters, facets,
+ index->search_params = new search_args(query, search_fields, filters, facets,
index_to_included_ids[index_id], index_to_excluded_ids[index_id],
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefix,
@@ -639,9 +639,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
index->cv.wait(lk, [index]{return index->processed;});
}
- if(!index->search_params.outcome.ok()) {
- index_search_op = Option<nlohmann::json>(index->search_params.outcome.code(),
- index->search_params.outcome.error());
+ if(!index->search_params->outcome.ok()) {
+ index_search_op = Option<nlohmann::json>(index->search_params->outcome.code(),
+ index->search_params->outcome.error());
}
if(!index_search_op.ok()) {
@@ -649,21 +649,21 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
continue;
}
- for(auto & field_order_kv: index->search_params.raw_result_kvs) {
- field_order_kv.query_index += searched_queries.size();
+ for(auto & field_order_kv: index->search_params->raw_result_kvs) {
+ field_order_kv->query_index += searched_queries.size();
raw_result_kvs.push_back(field_order_kv);
}
- for(auto & field_order_kv: index->search_params.override_result_kvs) {
- field_order_kv.query_index += searched_queries.size();
+ for(auto & field_order_kv: index->search_params->override_result_kvs) {
+ field_order_kv->query_index += searched_queries.size();
override_result_kvs.push_back(field_order_kv);
}
- searched_queries.insert(searched_queries.end(), index->search_params.searched_queries.begin(),
- index->search_params.searched_queries.end());
+ searched_queries.insert(searched_queries.end(), index->search_params->searched_queries.begin(),
+ index->search_params->searched_queries.end());
- for(size_t fi = 0; fi < index->search_params.facets.size(); fi++) {
- auto & this_facet = index->search_params.facets[fi];
+ for(size_t fi = 0; fi < index->search_params->facets.size(); fi++) {
+ auto & this_facet = index->search_params->facets[fi];
auto & acc_facet = facets[fi];
for(auto & facet_kv: this_facet.result_map) {
@@ -690,7 +690,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
}
- total_found += index->search_params.all_result_ids_len;
+ total_found += index->search_params->all_result_ids_len;
}
if(!index_search_op.ok()) {
@@ -698,13 +698,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
// All fields are sorted descending
- std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_value);
+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv);
- // Sort based on position in overriden list
+ // Sort based on position in overridden list
std::sort(
override_result_kvs.begin(), override_result_kvs.end(),
- [&id_pos_map](const KV & a, const KV & b) -> bool {
- return id_pos_map[a.key] < id_pos_map[b.key];
+ [&id_pos_map](const KV* a, const KV* b) -> bool {
+ return id_pos_map[a->key] < id_pos_map[b->key];
}
);
@@ -713,15 +713,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
result["hits"] = nlohmann::json::array();
result["found"] = total_found;
- std::vector<KV> result_kvs;
+ std::vector<KV*> result_kvs;
size_t override_kv_index = 0;
size_t raw_results_index = 0;
// merge raw results and override results
while(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {
if(override_kv_index < override_result_kvs.size() &&
- id_pos_map.count(override_result_kvs[override_kv_index].key) != 0 &&
- result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index].key]) {
+ id_pos_map.count(override_result_kvs[override_kv_index]->key) != 0 &&
+ result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index]->key]) {
result_kvs.push_back(override_result_kvs[override_kv_index]);
override_kv_index++;
} else {
@@ -746,7 +746,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// construct results array
for(long result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {
const auto & field_order_kv = result_kvs[result_kvs_index];
- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);
nlohmann::json document;
const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
@@ -818,8 +818,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
prune_document(document, include_fields, exclude_fields);
wrapper_doc["document"] = document;
- wrapper_doc["text_match"] = field_order_kv.match_score;
- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;
+ wrapper_doc["text_match"] = field_order_kv->match_score;
+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;
result["hits"].push_back(wrapper_doc);
}
@@ -918,6 +918,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
result["facet_counts"].push_back(facet_result);
}
+ // free search params
+ for(Index* index: indices) {
+ delete index->search_params;
+ }
+
result["request_params"] = nlohmann::json::object();;
result["request_params"]["per_page"] = per_page;
result["request_params"]["q"] = query;
@@ -966,7 +971,7 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
void Collection::highlight_result(const field &search_field,
const std::vector<std::vector<art_leaf *>> &searched_queries,
- const KV & field_order_kv, const nlohmann::json & document,
+ const KV* field_order_kv, const nlohmann::json & document,
StringUtils & string_utils, size_t snippet_threshold,
bool highlighted_fully,
highlight_t & highlight) {
@@ -974,15 +979,15 @@ void Collection::highlight_result(const field &search_field,
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
std::vector<art_leaf *> query_suggestion;
- for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
+ for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
// Must search for the token string fresh on that field for the given document since `token_leaf`
// is from the best matched field and need not be present in other fields of a document.
- Index* index = indices[field_order_kv.key % num_indices];
+ Index* index = indices[field_order_kv->key % num_indices];
art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);
if(actual_leaf != nullptr) {
query_suggestion.push_back(actual_leaf);
std::vector<uint16_t> positions;
- uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv.key);
+ uint32_t doc_index = actual_leaf->values->ids.indexOf(field_order_kv->key);
auto doc_indices = new uint32_t[1];
doc_indices[0] = doc_index;
leaf_to_indices.emplace(actual_leaf, doc_indices);
@@ -1008,8 +1013,8 @@ void Collection::highlight_result(const field &search_field,
continue;
}
- const Match & this_match = Match::match(field_order_kv.key, token_positions);
- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);
+ const Match & this_match = Match::match(field_order_kv->key, token_positions);
+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);
match_indices.emplace_back(this_match, this_match_score, array_index);
}
diff --git a/src/index.cpp b/src/index.cpp
index 9ce1b396..1ff8f5bb 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -829,7 +829,7 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>
void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,
const std::vector<sort_by> & sort_fields,
std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,
- std::vector<std::vector<art_leaf*>> & searched_queries, Topster & topster,
+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster* topster,
uint32_t** all_result_ids, size_t & all_result_ids_len,
const size_t typo_tokens_threshold) {
const long long combination_limit = 10;
@@ -1055,13 +1055,15 @@ void Index::run_search() {
}
// after the wait, we own the lock.
- search(search_params.outcome, search_params.query, search_params.search_fields,
- search_params.filters, search_params.facets, search_params.facet_query, search_params.included_ids,
- search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,
- search_params.max_hits, search_params.per_page, search_params.page, search_params.token_order,
- search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,
- search_params.all_result_ids_len, search_params.searched_queries, search_params.override_result_kvs,
- search_params.typo_tokens_threshold);
+ search(search_params->outcome, search_params->query, search_params->search_fields,
+ search_params->filters, search_params->facets, search_params->facet_query, search_params->included_ids,
+ search_params->excluded_ids, search_params->sort_fields_std, search_params->num_typos,
+ search_params->topster, search_params->curated_topster,
+ search_params->per_page, search_params->page, search_params->token_order,
+ search_params->prefix, search_params->drop_tokens_threshold,
+ search_params->all_result_ids_len, search_params->searched_queries,
+ search_params->raw_result_kvs, search_params->override_result_kvs,
+ search_params->typo_tokens_threshold);
// hand control back to main thread
processed = true;
@@ -1075,7 +1077,7 @@ void Index::run_search() {
void Index::collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,
const std::vector<uint32_t> & included_ids,
- Topster & curated_topster,
+ Topster* curated_topster,
std::vector<std::vector<art_leaf*>> & searched_queries) {
if(included_ids.size() == 0) {
@@ -1110,9 +1112,9 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
leaf_to_indices.emplace(token_leaf, indices);
}
- // curated_topster.MAX_SIZE is initialized based on max_hits.
+ // curated_topster->MAX_SIZE is initialized based on max_hits.
// Even if override has more IDs, we should restrict to max hits.
- size_t iter_size = std::min((size_t)curated_topster.MAX_SIZE, included_ids.size());
+ size_t iter_size = std::min((size_t)curated_topster->MAX_SIZE, included_ids.size());
for(size_t j=0; j<iter_size; j++) {
const uint32_t seq_id = included_ids[j];
@@ -1140,7 +1142,7 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
scores[2] = int64_t(1);
KV kv(field_id, searched_queries.size(), seq_id, seq_id, match_score, scores);
- curated_topster.add(&kv);
+ curated_topster->add(&kv);
searched_queries.push_back(override_query);
}
@@ -1153,13 +1155,15 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<facet> & facets, facet_query_t & facet_query,
const std::vector<uint32_t> & included_ids,
const std::vector<uint32_t> & excluded_ids,
- const std::vector<sort_by> & sort_fields_std, const int num_typos, const size_t max_hits,
+ const std::vector<sort_by> & sort_fields_std, const int num_typos,
+ Topster* topster,
+ Topster* curated_topster,
const size_t per_page, const size_t page, const token_ordering token_order,
const bool prefix, const size_t drop_tokens_threshold,
- std::vector<KV> & raw_result_kvs,
size_t & all_result_ids_len,
std::vector<std::vector<art_leaf*>> & searched_queries,
- std::vector<KV> & override_result_kvs,
+ std::vector<KV*> & raw_result_kvs,
+ std::vector<KV*> & override_result_kvs,
const size_t typo_tokens_threshold) {
const size_t num_results = (page * per_page);
@@ -1179,10 +1183,6 @@ void Index::search(Option<uint32_t> & outcome,
//auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;
- const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
- Topster topster(topster_size);
- Topster curated_topster(topster_size);
-
if(query == "*") {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string & field = search_fields[0];
@@ -1212,8 +1212,8 @@ void Index::search(Option<uint32_t> & outcome,
do_facets(facets, facet_query, &included_ids[0], included_ids.size());
// must be sorted before iterated upon to remove "empty" array entries
- topster.sort();
- curated_topster.sort();
+ topster->sort();
+ curated_topster->sort();
std::set<uint32_t> ids_to_remove(included_ids.begin(), included_ids.end());
ids_to_remove.insert(excluded_ids.begin(), excluded_ids.end());
@@ -1221,26 +1221,26 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<uint32_t> dropped_ids;
// loop through topster and remove elements from included and excluded id lists
- for(uint32_t t = 0; t < topster.size && t < num_results; t++) {
- KV* kv = topster.getKV(t);
+ for(uint32_t t = 0; t < topster->size && t < num_results; t++) {
+ KV* kv = topster->getKV(t);
if(ids_to_remove.count(kv->key) != 0) {
dropped_ids.push_back((uint32_t)kv->key);
} else {
- raw_result_kvs.push_back(*kv);
+ raw_result_kvs.push_back(kv);
}
}
- for(uint32_t t = 0; t < curated_topster.size && t < num_results; t++) {
- KV* kv = curated_topster.getKV(t);
- override_result_kvs.push_back(*kv);
+ for(uint32_t t = 0; t < curated_topster->size && t < num_results; t++) {
+ KV* kv = curated_topster->getKV(t);
+ override_result_kvs.push_back(kv);
}
// for the ids that are dropped, remove their corresponding facet components from facet results
drop_facets(facets, dropped_ids);
all_result_ids_len -= dropped_ids.size();
- all_result_ids_len += curated_topster.size;
+ all_result_ids_len += curated_topster->size;
delete [] filter_ids;
delete [] all_result_ids;
@@ -1264,7 +1264,7 @@ void Index::search_field(const uint8_t & field_id, const std::string & query, co
uint32_t *filter_ids, size_t filter_ids_length,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
std::vector<std::vector<art_leaf*>> & searched_queries,
- Topster & topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ Topster* topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
const token_ordering token_order, const bool prefix,
const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) {
std::vector<std::string> tokens;
@@ -1438,7 +1438,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
- const uint8_t & field_id, const uint32_t total_cost, Topster & topster,
+ const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const {
@@ -1546,7 +1546,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
KV kv(field_id, query_index, seq_id, seq_id, match_score, scores);
- topster.add(&kv);
+ topster->add(&kv);
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();