Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor topster to support grouping.
kishorenc5b2407433f46
diff --git a/include/topster.h b/include/topster.hindex 6ebadece..a065bb01 100644--- a/include/topster.h+++ b/include/topster.h@@ -5,16 +5,26 @@#include <cstdio>#include <algorithm>#include <sparsepp.h>-#include <match_score.h>-#include <number.h>struct KV {uint8_t field_id;uint16_t query_index;uint16_t array_index;uint64_t key;+ uint64_t distinct_key;uint64_t match_score;- int64_t scores[3]; // match score + 2 custom attributes+ int64_t scores[3]{}; // match score + 2 custom attributes++ KV(uint8_t fieldId, uint16_t queryIndex, uint16_t arrayIndex, uint64_t key, uint64_t distinct_key,+ uint64_t match_score, const int64_t *scores):+ field_id(fieldId), query_index(queryIndex), array_index(arrayIndex), key(key),+ distinct_key(distinct_key), match_score(match_score) {+ this->scores[0] = scores[0];+ this->scores[1] = scores[1];+ this->scores[2] = scores[2];+ }++ KV() {}};/*@@ -25,12 +35,18 @@ struct Topster {uint32_t size;KV *data;- KV* *kvs;+ KV** kvs;+ spp::sparse_hash_map<uint64_t, KV*> kv_map;- spp::sparse_hash_map<uint64_t, KV*> keys;+ KV* min_kv;+ spp::sparse_hash_map<uint64_t, Topster*> group_kv_map;+ size_t distinct;++ explicit Topster(size_t capacity): Topster(capacity, 0) {+ }- explicit Topster(size_t capacity): MAX_SIZE(capacity), size(0) {- // we allocate data first to get contiguous memory block whose indices are then assigned to `kvs`+ explicit Topster(size_t capacity, size_t distinct): MAX_SIZE(capacity), size(0), distinct(distinct) {+ // we allocate data first to get a memory block whose indices are then assigned to `kvs`// we use separate **kvs for easier pointer swapsdata = new KV[capacity];kvs = new KV*[capacity];@@ -38,15 +54,23 @@ struct Topster {for(size_t i=0; i<capacity; i++) {data[i].field_id = 0;data[i].query_index = 0;+ data[i].array_index = i;data[i].key = 0;+ data[i].distinct_key = 0;data[i].match_score = 0;kvs[i] = &data[i];}++ min_kv = new KV();}~Topster() {- delete [] data;- delete [] kvs;+ delete[] data;+ delete[] kvs;+ delete min_kv;+ for(auto& kv: group_kv_map) {+ delete kv.second;+ }}static inline void swapMe(KV** a, KV** b) {@@ -59,118 +83,137 @@ struct Topster {(*b)->array_index = a_index;}- static inline void replace_key_values(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index,- const uint64_t &match_score, const int64_t *scores, uint32_t start,- KV* *kvs, spp::sparse_hash_map<uint64_t, KV*>& keys) {- kvs[start]->key = key;- kvs[start]->field_id = field_id;- kvs[start]->query_index = query_index;- kvs[start]->array_index = start;- kvs[start]->match_score = match_score;- kvs[start]->scores[0] = scores[0];- kvs[start]->scores[1] = scores[1];- kvs[start]->scores[2] = scores[2];-- keys.erase(kvs[start]->key);- keys[key] = kvs[start];+ static inline void copyMe(KV* a, KV* b) {+ size_t b_index = b->array_index;+ *b = *a;+ b->array_index = b_index;}- void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,- const int64_t scores[3]) {- if (size >= MAX_SIZE) {- if(!is_greater(kvs[0], scores)) {- // when incoming value is less than the smallest in the heap, ignore- return;- }+ bool add(KV* kv) {+ //LOG(INFO) << "kv_map size: " << kv_map.size() << " -- kvs[0]: " << kvs[0]->match_score;+ /*for(auto kv: kv_map) {+ LOG(INFO) << "kv key: " << kv.first << " => " << kv.second->match_score;+ }*/- uint32_t start = 0;+ bool less_than_min_heap = (size >= MAX_SIZE) && is_smaller_equal(kv, kvs[0]);+ size_t heap_down_index = 0;- // When the key already exists and has a greater score, ignore. Otherwise, we have to replace.- // NOTE: we don't consider primary and secondary attrs here because they will be the same for a given key.- if(keys.count(key) != 0) {- const KV* existing = keys.at(key);- if(match_score <= existing->match_score) {- return ;- }+ if(!distinct && less_than_min_heap) {+ // for non-distinct, if incoming value is smaller than min-heap ignore+ return false;+ }- // replace and sift down- start = existing->array_index;- }+ if(distinct) {+ const auto& found_it = group_kv_map.find(kv->distinct_key);+ bool is_duplicate_key = (found_it != group_kv_map.end());- replace_key_values(key, field_id, query_index, match_score, scores, start, kvs, keys);+ if(!is_duplicate_key && less_than_min_heap) {+ // for distinct, if a non duplicate kv is < than min heap we also ignore+ return false;+ }- // sift down to maintain heap property- while ((2*start+1) < MAX_SIZE) {- uint32_t next = (2 * start + 1);- if (next+1 < MAX_SIZE && is_greater_kv(kvs[next], kvs[next+1])) {- next++;+ if(is_duplicate_key) {+ // if min heap (group_topster.kvs[0]) changes, we have to update kvs and sift down+ Topster* group_topster = found_it->second;+ uint16_t old_min_heap_array_index = group_topster->min_kv->array_index;+ bool added = group_topster->add(kv);+ if(!added) {+ return false;}- if (is_greater_kv(kvs[start], kvs[next])) {- swapMe(&kvs[start], &kvs[next]);+ // if added, guaranteed to be larger than old_min_heap_ele+ copyMe(kv, group_topster->min_kv);+ heap_down_index = old_min_heap_array_index;+ } else {+ // we have to replace min heap element+ // create fresh topster for this distinct group key since it does not exist++ Topster* group_topster = new Topster(distinct, 0);+ group_topster->add(kv);+ copyMe(kv, group_topster->min_kv);++ if(size < MAX_SIZE) {+ // we just copy to end of array+ heap_down_index = size;+ size++;} else {- break;- }+ // kv is guaranteed to be > current min heap (group_topster.kvs[0])+ heap_down_index = 0;- start = next;- }- } else {- uint32_t start = size;- bool key_found = false;-- // When the key already exists and has a greater score, ignore. Otherwise, we have to replace- if(keys.count(key) != 0) {- const KV* existing = keys.at(key);- if(match_score <= existing->match_score) {- return ;+ // remove current min heap group key from map+ delete group_kv_map[kvs[heap_down_index]->distinct_key];+ group_kv_map.erase(kvs[heap_down_index]->distinct_key);}- // replace and sift down- start = existing->array_index;- key_found = true;+ // add new group key to map+ group_kv_map.emplace(kv->distinct_key, group_topster);}- replace_key_values(key, field_id, query_index, match_score, scores, start, kvs, keys);+ } else { // not distinct+ //LOG(INFO) << "Searching for key: " << kv->key;- if(key_found) {- // need to sift down if it's a replace- while ((2*start+1) < size) {- uint32_t next = (2 * start + 1);- if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {- next++;- }+ const auto& found_it = kv_map.find(kv->key);+ bool is_duplicate_key = (found_it != kv_map.end());- if (is_greater_kv(kvs[start], kvs[next])) {- swapMe(&kvs[start], &kvs[next]);- } else {- break;- }+ if(is_duplicate_key) {+ // Need to check if kv is greater than existing duplicate kv.+ KV* existing_kv = found_it->second;+ //LOG(INFO) << "existing_kv: " << existing_kv->key << " -> " << existing_kv->match_score;- start = next;+ if(is_smaller_equal(kv, existing_kv)) {+ return false;}- return ;+ // replace existing kv and sift down+ heap_down_index = existing_kv->array_index;+ kv_map.erase(kvs[heap_down_index]->key);++ // kv will be swapped into heap_down_index+ kv_map.emplace(kv->key, kvs[heap_down_index]);}- while(start > 0) {- uint32_t parent = (start-1)/2;- if (is_greater_kv(kvs[parent], kvs[start])) {- swapMe(&kvs[start], &kvs[parent]);- start = parent;+ else {+ if(size < MAX_SIZE) {+ // we just copy to end of array+ heap_down_index = size;+ size++;} else {- break;+ // kv is guaranteed to be > min heap.+ // we have to replace min heap element since array is full+ heap_down_index = 0;+ kv_map.erase(kvs[heap_down_index]->key);}++ // kv will be swapped into heap_down_index pointer+ kv_map.emplace(kv->key, kvs[heap_down_index]);+ }+ }++ // we have to replace the existing element in the heap and sift down+ copyMe(kv, kvs[heap_down_index]);++ if(size < MAX_SIZE) {+ heap_down_index = 0;+ }++ // sift down to maintain heap property+ while ((2*heap_down_index+1) < size) {+ uint32_t next = (2 * heap_down_index + 1); // left child+ if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {+ // for min heap we compare with the minimum of children+ next++; // right child (2n + 2)}- if(keys.count(key) != 0) {- size++;+ if (is_greater_kv(kvs[heap_down_index], kvs[next])) {+ swapMe(&kvs[heap_down_index], &kvs[next]);+ } else {+ break;}++ heap_down_index = next;}- }- static bool is_greater(const struct KV* i, const int64_t scores[3]) {- return std::tie(scores[0], scores[1], scores[2]) >- std::tie(i->scores[0], i->scores[1], i->scores[2]);+ return true;}static bool is_greater_kv(const struct KV* i, const struct KV* j) {@@ -178,6 +221,11 @@ struct Topster {std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);}+ static bool is_smaller_equal(const struct KV* i, const struct KV* j) {+ return std::tie(i->scores[0], i->scores[1], i->scores[2]) <=+ std::tie(j->scores[0], j->scores[1], j->scores[2]);+ }+static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);diff --git a/src/index.cpp b/src/index.cppindex 71dd092b..96df1f78 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1135,7 +1135,8 @@ void Index::collate_curated_ids(const std::string & query, const std::string & fscores[1] = int64_t(1);scores[2] = int64_t(1);- curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);+ KV kv(field_id, searched_queries.size(), 0, seq_id, seq_id, match_score, scores);+ curated_topster.add(&kv);searched_queries.push_back(override_query);}@@ -1540,7 +1541,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}}- topster.add(seq_id, field_id, query_index, match_score, scores);+ KV kv(field_id, query_index, 0, seq_id, seq_id, match_score, scores);+ topster.add(&kv);}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex 793eb00a..3000c26e 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -46,7 +46,7 @@ void benchmark_hn_titles(char* file_path) {Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(store, 4, "abcd", "1234");+ collectionManager.init(store, 4, "abcd");collectionManager.load();Collection *collection = collectionManager.get_collection("hnstories_direct");@@ -116,7 +116,7 @@ void benchmark_reactjs_pages(char* file_path) {Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(store, 4, "abcd", "1234");+ collectionManager.init(store, 4, "abcd");collectionManager.load();Collection *collection = collectionManager.get_collection("reactjs_pages");diff --git a/test/topster_test.cpp b/test/topster_test.cppindex 1e9438a8..a685fcb0 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -36,7 +36,8 @@ TEST(TopsterTest, MaxIntValues) {scores[1] = data[i].primary_attr;scores[2] = data[i].secondary_attr;- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);+ KV kv(data[i].field_id, data[i].query_index, 0, data[i].key, data[i].key, data[i].match_score, scores);+ topster.add(&kv);}topster.sort();@@ -87,7 +88,8 @@ TEST(TopsterTest, MaxFloatValues) {scores[1] = Index::float_to_in64_t(data[i].primary_attr);scores[2] = data[i].secondary_attr;- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);+ KV kv(data[i].field_id, data[i].query_index, 0, data[i].key, data[i].key, data[i].match_score, scores);+ topster.add(&kv);}topster.sort();