Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor topster to support grouping.

kishorenc5b2407433f46
diff --git a/include/topster.h b/include/topster.h
index 6ebadece..a065bb01 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -5,16 +5,26 @@
#include <cstdio>
#include <algorithm>
#include <sparsepp.h>
-#include <match_score.h>
-#include <number.h>
struct KV {
uint8_t field_id;
uint16_t query_index;
uint16_t array_index;
uint64_t key;
+ uint64_t distinct_key;
uint64_t match_score;
- int64_t scores[3]; // match score + 2 custom attributes
+ int64_t scores[3]{}; // match score + 2 custom attributes
+
+ KV(uint8_t fieldId, uint16_t queryIndex, uint16_t arrayIndex, uint64_t key, uint64_t distinct_key,
+ uint64_t match_score, const int64_t *scores):
+ field_id(fieldId), query_index(queryIndex), array_index(arrayIndex), key(key),
+ distinct_key(distinct_key), match_score(match_score) {
+ this->scores[0] = scores[0];
+ this->scores[1] = scores[1];
+ this->scores[2] = scores[2];
+ }
+
+ KV() {}
};
/*
@@ -25,12 +35,18 @@ struct Topster {
uint32_t size;
KV *data;
- KV* *kvs;
+ KV** kvs;
+ spp::sparse_hash_map<uint64_t, KV*> kv_map;
- spp::sparse_hash_map<uint64_t, KV*> keys;
+ KV* min_kv;
+ spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;
+ size_t distinct;
+
+ explicit Topster(size_t capacity): Topster(capacity, 0) {
+ }
- explicit Topster(size_t capacity): MAX_SIZE(capacity), size(0) {
- // we allocate data first to get contiguous memory block whose indices are then assigned to `kvs`
+ explicit Topster(size_t capacity, size_t distinct): MAX_SIZE(capacity), size(0), distinct(distinct) {
+ // we allocate data first to get a memory block whose indices are then assigned to `kvs`
// we use separate **kvs for easier pointer swaps
data = new KV[capacity];
kvs = new KV*[capacity];
@@ -38,15 +54,23 @@ struct Topster {
for(size_t i=0; i<capacity; i++) {
data[i].field_id = 0;
data[i].query_index = 0;
+ data[i].array_index = i;
data[i].key = 0;
+ data[i].distinct_key = 0;
data[i].match_score = 0;
kvs[i] = &data[i];
}
+
+ min_kv = new KV();
}
~Topster() {
- delete [] data;
- delete [] kvs;
+ delete[] data;
+ delete[] kvs;
+ delete min_kv;
+ for(auto& kv: group_kv_map) {
+ delete kv.second;
+ }
}
static inline void swapMe(KV** a, KV** b) {
@@ -59,118 +83,137 @@ struct Topster {
(*b)->array_index = a_index;
}
- static inline void replace_key_values(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index,
- const uint64_t &match_score, const int64_t *scores, uint32_t start,
- KV* *kvs, spp::sparse_hash_map<uint64_t, KV*>& keys) {
- kvs[start]->key = key;
- kvs[start]->field_id = field_id;
- kvs[start]->query_index = query_index;
- kvs[start]->array_index = start;
- kvs[start]->match_score = match_score;
- kvs[start]->scores[0] = scores[0];
- kvs[start]->scores[1] = scores[1];
- kvs[start]->scores[2] = scores[2];
-
- keys.erase(kvs[start]->key);
- keys[key] = kvs[start];
+ static inline void copyMe(KV* a, KV* b) {
+ size_t b_index = b->array_index;
+ *b = *a;
+ b->array_index = b_index;
}
- void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,
- const int64_t scores[3]) {
- if (size >= MAX_SIZE) {
- if(!is_greater(kvs[0], scores)) {
- // when incoming value is less than the smallest in the heap, ignore
- return;
- }
+ bool add(KV* kv) {
+ //LOG(INFO) << "kv_map size: " << kv_map.size() << " -- kvs[0]: " << kvs[0]->match_score;
+ /*for(auto kv: kv_map) {
+ LOG(INFO) << "kv key: " << kv.first << " => " << kv.second->match_score;
+ }*/
- uint32_t start = 0;
+ bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller_equal(kv, kvs[0]);
+ size_t heap_down_index = 0;
- // When the key already exists and has a greater score, ignore. Otherwise, we have to replace.
- // NOTE: we don't consider primary and secondary attrs here because they will be the same for a given key.
- if(keys.count(key) != 0) {
- const KV* existing = keys.at(key);
- if(match_score <= existing->match_score) {
- return ;
- }
+ if(!distinct && less_than_min_heap) {
+ // for non-distinct, if incoming value is smaller than min-heap ignore
+ return false;
+ }
- // replace and sift down
- start = existing->array_index;
- }
+ if(distinct) {
+ const auto& found_it = group_kv_map.find(kv->distinct_key);
+ bool is_duplicate_key = (found_it != group_kv_map.end());
- replace_key_values(key, field_id, query_index, match_score, scores, start, kvs, keys);
+ if(!is_duplicate_key && less_than_min_heap) {
+ // for distinct, if a non duplicate kv is < than min heap we also ignore
+ return false;
+ }
- // sift down to maintain heap property
- while ((2*start+1) < MAX_SIZE) {
- uint32_t next = (2 * start + 1);
- if (next+1 < MAX_SIZE && is_greater_kv(kvs[next], kvs[next+1])) {
- next++;
+ if(is_duplicate_key) {
+ // if min heap (group_topster.kvs[0]) changes, we have to update kvs and sift down
+ Topster* group_topster = found_it->second;
+ uint16_t old_min_heap_array_index = group_topster->min_kv->array_index;
+ bool added = group_topster->add(kv);
+ if(!added) {
+ return false;
}
- if (is_greater_kv(kvs[start], kvs[next])) {
- swapMe(&kvs[start], &kvs[next]);
+ // if added, guaranteed to be larger than old_min_heap_ele
+ copyMe(kv, group_topster->min_kv);
+ heap_down_index = old_min_heap_array_index;
+ } else {
+ // we have to replace min heap element
+ // create fresh topster for this distinct group key since it does not exist
+
+ Topster* group_topster = new Topster(distinct, 0);
+ group_topster->add(kv);
+ copyMe(kv, group_topster->min_kv);
+
+ if(size < MAX_SIZE) {
+ // we just copy to end of array
+ heap_down_index = size;
+ size++;
} else {
- break;
- }
+ // kv is guaranteed to be > current min heap (group_topster.kvs[0])
+ heap_down_index = 0;
- start = next;
- }
- } else {
- uint32_t start = size;
- bool key_found = false;
-
- // When the key already exists and has a greater score, ignore. Otherwise, we have to replace
- if(keys.count(key) != 0) {
- const KV* existing = keys.at(key);
- if(match_score <= existing->match_score) {
- return ;
+ // remove current min heap group key from map
+ delete group_kv_map[kvs[heap_down_index]->distinct_key];
+ group_kv_map.erase(kvs[heap_down_index]->distinct_key);
}
- // replace and sift down
- start = existing->array_index;
- key_found = true;
+ // add new group key to map
+ group_kv_map.emplace(kv->distinct_key, group_topster);
}
- replace_key_values(key, field_id, query_index, match_score, scores, start, kvs, keys);
+ } else { // not distinct
+ //LOG(INFO) << "Searching for key: " << kv->key;
- if(key_found) {
- // need to sift down if it's a replace
- while ((2*start+1) < size) {
- uint32_t next = (2 * start + 1);
- if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {
- next++;
- }
+ const auto& found_it = kv_map.find(kv->key);
+ bool is_duplicate_key = (found_it != kv_map.end());
- if (is_greater_kv(kvs[start], kvs[next])) {
- swapMe(&kvs[start], &kvs[next]);
- } else {
- break;
- }
+ if(is_duplicate_key) {
+ // Need to check if kv is greater than existing duplicate kv.
+ KV* existing_kv = found_it->second;
+ //LOG(INFO) << "existing_kv: " << existing_kv->key << " -> " << existing_kv->match_score;
- start = next;
+ if(is_smaller_equal(kv, existing_kv)) {
+ return false;
}
- return ;
+ // replace existing kv and sift down
+ heap_down_index = existing_kv->array_index;
+ kv_map.erase(kvs[heap_down_index]->key);
+
+ // kv will be swapped into heap_down_index
+ kv_map.emplace(kv->key, kvs[heap_down_index]);
}
- while(start > 0) {
- uint32_t parent = (start-1)/2;
- if (is_greater_kv(kvs[parent], kvs[start])) {
- swapMe(&kvs[start], &kvs[parent]);
- start = parent;
+ else {
+ if(size < MAX_SIZE) {
+ // we just copy to end of array
+ heap_down_index = size;
+ size++;
} else {
- break;
+ // kv is guaranteed to be > min heap.
+ // we have to replace min heap element since array is full
+ heap_down_index = 0;
+ kv_map.erase(kvs[heap_down_index]->key);
}
+
+ // kv will be swapped into heap_down_index pointer
+ kv_map.emplace(kv->key, kvs[heap_down_index]);
+ }
+ }
+
+ // we have to replace the existing element in the heap and sift down
+ copyMe(kv, kvs[heap_down_index]);
+
+ if(size < MAX_SIZE) {
+ heap_down_index = 0;
+ }
+
+ // sift down to maintain heap property
+ while ((2*heap_down_index+1) < size) {
+ uint32_t next = (2 * heap_down_index + 1); // left child
+ if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {
+ // for min heap we compare with the minimum of children
+ next++; // right child (2n + 2)
}
- if(keys.count(key) != 0) {
- size++;
+ if (is_greater_kv(kvs[heap_down_index], kvs[next])) {
+ swapMe(&kvs[heap_down_index], &kvs[next]);
+ } else {
+ break;
}
+
+ heap_down_index = next;
}
- }
- static bool is_greater(const struct KV* i, const int64_t scores[3]) {
- return std::tie(scores[0], scores[1], scores[2]) >
- std::tie(i->scores[0], i->scores[1], i->scores[2]);
+ return true;
}
static bool is_greater_kv(const struct KV* i, const struct KV* j) {
@@ -178,6 +221,11 @@ struct Topster {
std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);
}
+ static bool is_smaller_equal(const struct KV* i, const struct KV* j) {
+ return std::tie(i->scores[0], i->scores[1], i->scores[2]) <=
+ std::tie(j->scores[0], j->scores[1], j->scores[2]);
+ }
+
static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {
return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >
std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);
diff --git a/src/index.cpp b/src/index.cpp
index 71dd092b..96df1f78 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1135,7 +1135,8 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
scores[1] = int64_t(1);
scores[2] = int64_t(1);
- curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);
+ KV kv(field_id, searched_queries.size(), 0, seq_id, seq_id, match_score, scores);
+ curated_topster.add(&kv);
searched_queries.push_back(override_query);
}
@@ -1540,7 +1541,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
}
- topster.add(seq_id, field_id, query_index, match_score, scores);
+ KV kv(field_id, query_index, 0, seq_id, seq_id, match_score, scores);
+ topster.add(&kv);
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index 793eb00a..3000c26e 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -46,7 +46,7 @@ void benchmark_hn_titles(char* file_path) {
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(store, 4, "abcd", "1234");
+ collectionManager.init(store, 4, "abcd");
collectionManager.load();
Collection *collection = collectionManager.get_collection("hnstories_direct");
@@ -116,7 +116,7 @@ void benchmark_reactjs_pages(char* file_path) {
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(store, 4, "abcd", "1234");
+ collectionManager.init(store, 4, "abcd");
collectionManager.load();
Collection *collection = collectionManager.get_collection("reactjs_pages");
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 1e9438a8..a685fcb0 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -36,7 +36,8 @@ TEST(TopsterTest, MaxIntValues) {
scores[1] = data[i].primary_attr;
scores[2] = data[i].secondary_attr;
- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
+ KV kv(data[i].field_id, data[i].query_index, 0, data[i].key, data[i].key, data[i].match_score, scores);
+ topster.add(&kv);
}
topster.sort();
@@ -87,7 +88,8 @@ TEST(TopsterTest, MaxFloatValues) {
scores[1] = Index::float_to_in64_t(data[i].primary_attr);
scores[2] = data[i].secondary_attr;
- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
+ KV kv(data[i].field_id, data[i].query_index, 0, data[i].key, data[i].key, data[i].match_score, scores);
+ topster.add(&kv);
}
topster.sort();