Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use heap allocation rather than stack allocation for topster data.

Kishore Nallan4f1736124bd1
diff --git a/TODO.md b/TODO.md
index ba6b6966..87cd8c58 100644
--- a/TODO.md
+++ b/TODO.md
@@ -96,6 +96,7 @@
- ~~gzip compress responses~~
- ~~Have a LOG(ERROR) level~~
- ~~Handle SIGTERM which is sent when process is killed~~
+- Use snappy compression for storage
- Exact search
- NOT operator support
- Log operations
diff --git a/include/index.h b/include/index.h
index f5351647..26cc9f42 100644
--- a/include/index.h
+++ b/include/index.h
@@ -33,7 +33,7 @@ struct search_args {
token_ordering token_order;
bool prefix;
size_t drop_tokens_threshold;
- std::vector<Topster<512>::KV*> field_order_kvs;
+ std::vector<Topster<512>::KV> field_order_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
Option<uint32_t> outcome;
@@ -142,7 +142,7 @@ public:
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV*> & field_order_kv,
+ std::vector<Topster<512>::KV> & field_order_kv,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
diff --git a/include/string_utils.h b/include/string_utils.h
index 0dfcfc9d..5672469a 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -9,14 +9,14 @@
struct StringUtils {
UErrorCode status;
- icu::Transliterator* transliterator;
+ //icu::Transliterator* transliterator;
- StringUtils():status(U_ZERO_ERROR),
- transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status)) {
+ StringUtils(): status(U_ZERO_ERROR) {
+ // transliterator(icu::Transliterator::createInstance("Latin-ASCII", UTRANS_FORWARD, status))
}
~StringUtils() {
- delete transliterator;
+ //delete transliterator;
}
// Adapted from: http://stackoverflow.com/a/236180/131050
diff --git a/include/topster.h b/include/topster.h
index 7d3e87c3..2f803c82 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -21,8 +21,9 @@ struct Topster {
uint64_t match_score;
number_t primary_attr;
number_t secondary_attr;
- } data[MAX_SIZE];
+ };
+ KV *data;
uint32_t size;
spp::sparse_hash_map<uint64_t, KV*> keys;
@@ -30,12 +31,23 @@ struct Topster {
KV *kvs[MAX_SIZE];
Topster(): size(0){
+ data = new KV[MAX_SIZE];
+
for(size_t i=0; i<MAX_SIZE; i++) {
+ data[i].field_id = 0;
+ data[i].query_index = 0;
+ data[i].key = 0;
+ data[i].match_score = 0;
+ data[i].primary_attr = number_t();
+ data[i].secondary_attr = number_t();
kvs[i] = &data[i];
- kvs[i]->array_index = i;
}
}
+ ~Topster() {
+ delete [] data;
+ }
+
static inline void swapMe(KV** a, KV** b) {
KV *temp = *a;
*a = *b;
@@ -166,6 +178,11 @@ struct Topster {
std::tie(j->match_score, j->primary_attr, j->secondary_attr, j->key);
}
+ static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {
+ return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.key) >
+ std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.key);
+ }
+
// topster must be sorted before iterated upon to remove dead array entries
void sort() {
std::stable_sort(std::begin(kvs), std::begin(kvs) + size, is_greater_kv);
diff --git a/src/collection.cpp b/src/collection.cpp
index 8f665f86..855860be 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -428,7 +428,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// all search queries that were used for generating the results
std::vector<std::vector<art_leaf*>> searched_queries;
- std::vector<Topster<512>::KV*> field_order_kvs;
+ std::vector<Topster<512>::KV> field_order_kvs;
size_t total_found = 0;
// send data to individual index threads
@@ -463,8 +463,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- for(auto* field_order_kv: index->search_params.field_order_kvs) {
- field_order_kv->query_index += searched_queries.size();
+ for(auto & field_order_kv: index->search_params.field_order_kvs) {
+ field_order_kv.query_index += searched_queries.size();
field_order_kvs.push_back(field_order_kv);
}
@@ -497,7 +497,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
// All fields are sorted descending
- std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv);
+ std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv_value);
nlohmann::json result = nlohmann::json::object();
@@ -515,8 +515,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// construct results array
for(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {
- const auto* field_order_kv = field_order_kvs[field_order_kv_index];
- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);
+ const auto & field_order_kv = field_order_kvs[field_order_kv_index];
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);
std::string json_doc_str;
StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
@@ -536,19 +536,19 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
wrapper_doc["document"] = document;
- //wrapper_doc["match_score"] = field_order_kv->match_score;
- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;
+ //wrapper_doc["match_score"] = field_order_kv.match_score;
+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;
// highlight query words in the result
- const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv->field_id];
+ const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];
field search_field = search_schema.at(field_name);
if(search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
- for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
+ for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
std::vector<uint16_t> positions;
- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv->key);
+ uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);
uint32_t *indices = new uint32_t[1];
indices[0] = doc_index;
leaf_to_indices.emplace(token_leaf, indices);
@@ -556,7 +556,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// positions in the field of each token in the query
std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
- Index::populate_token_positions(searched_queries[field_order_kv->query_index],
+ Index::populate_token_positions(searched_queries[field_order_kv.query_index],
leaf_to_indices, 0, array_token_positions);
Match match;
@@ -570,8 +570,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- const Match & this_match = Match::match(field_order_kv->key, token_positions);
- uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);
+ const Match & this_match = Match::match(field_order_kv.key, token_positions);
+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv.field_id);
if(this_match_score > match_score) {
match_score = this_match_score;
match = this_match;
diff --git a/src/index.cpp b/src/index.cpp
index 83ebd74e..c1bff470 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -600,7 +600,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page, const token_ordering token_order,
const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV*> & field_order_kvs,
+ std::vector<Topster<512>::KV> & field_order_kvs,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {
const size_t num_results = (page * per_page);
@@ -640,7 +640,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
// order of fields specified matter: matching docs from earlier fields are more important
for(uint32_t t = 0; t < topster.size && t < num_results; t++) {
Topster<512>::KV* kv = topster.getKV(t);
- field_order_kvs.push_back(kv);
+ field_order_kvs.push_back(*kv);
}
delete [] filter_ids;