Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor topster to ensure that it handles insertion of duplicate keys.

Kishore Nallan56ed39e3ff1a
diff --git a/include/index.h b/include/index.h
index 26cc9f42..f5351647 100644
--- a/include/index.h
+++ b/include/index.h
@@ -33,7 +33,7 @@ struct search_args {
token_ordering token_order;
bool prefix;
size_t drop_tokens_threshold;
- std::vector<Topster<512>::KV> field_order_kvs;
+ std::vector<Topster<512>::KV*> field_order_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
Option<uint32_t> outcome;
@@ -142,7 +142,7 @@ public:
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV> & field_order_kv,
+ std::vector<Topster<512>::KV*> & field_order_kv,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
diff --git a/include/match_score.h b/include/match_score.h
index 29e85329..68fc529b 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -16,7 +16,7 @@
#define TokenOffsetHeap std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset>
const size_t WINDOW_SIZE = 10;
-const uint16_t MAX_DISPLACEMENT = std::numeric_limits<uint16_t>::max();
+const uint8_t MAX_DISPLACEMENT = std::numeric_limits<uint8_t>::max();
const uint16_t MAX_TOKENS_DISTANCE = 100;
struct TokenOffset {
@@ -30,8 +30,8 @@ struct TokenOffset {
};
struct Match {
- uint16_t words_present;
- uint16_t distance;
+ uint8_t words_present;
+ uint8_t distance;
uint16_t start_offset;
char offset_diffs[16];
@@ -39,16 +39,17 @@ struct Match {
}
- Match(uint16_t words_present, uint16_t distance, uint16_t start_offset, char *offset_diffs_stacked):
+ Match(uint8_t words_present, uint8_t distance, uint16_t start_offset, char *offset_diffs_stacked):
words_present(words_present), distance(distance), start_offset(start_offset) {
memcpy(offset_diffs, offset_diffs_stacked, 16);
}
// Construct a single match score from individual components (for multi-field sort)
- inline uint64_t get_match_score(const uint32_t total_cost) const {
- uint64_t match_score = ((int64_t)(words_present) << 24) |
+ inline uint64_t get_match_score(const uint32_t total_cost, const uint8_t field_id) const {
+ uint64_t match_score = ((int64_t)(words_present) << 24) |
((int64_t)(255 - total_cost) << 16) |
- ((int64_t)(distance));
+ ((int64_t)(distance) << 8) |
+ ((int64_t)(field_id));
return match_score;
}
@@ -109,7 +110,7 @@ struct Match {
// heap now contains the first occurring offset of each token in the given document
uint16_t max_match = 0;
- uint16_t min_displacement = MAX_DISPLACEMENT;
+ uint8_t min_displacement = MAX_DISPLACEMENT;
std::queue<TokenOffset> window;
uint16_t token_offset[WINDOW_SIZE] = { };
@@ -188,7 +189,7 @@ struct Match {
token_index++;
}
- const uint16_t distance = MAX_TOKENS_DISTANCE - min_displacement;
+ const uint8_t distance = MAX_TOKENS_DISTANCE - min_displacement;
pack_token_offsets(min_token_offset, token_offsets.size(), token_start_offset, packed_offset_diffs);
return Match(max_match, distance, token_start_offset, packed_offset_diffs);
}
diff --git a/include/topster.h b/include/topster.h
index 759f1714..7d3e87c3 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -16,6 +16,7 @@ struct Topster {
struct KV {
uint8_t field_id;
uint16_t query_index;
+ uint16_t array_index;
uint64_t key;
uint64_t match_score;
number_t primary_attr;
@@ -24,98 +25,150 @@ struct Topster {
uint32_t size;
- spp::sparse_hash_set<uint64_t> dedup_keys;
+ spp::sparse_hash_map<uint64_t, KV*> keys;
- Topster(): size(0){
+ KV *kvs[MAX_SIZE];
+ Topster(): size(0){
+ for(size_t i=0; i<MAX_SIZE; i++) {
+ kvs[i] = &data[i];
+ kvs[i]->array_index = i;
+ }
}
- template <typename T> static inline void swapMe(T& a, T& b) {
- T c = a;
- a = b;
- b = c;
+ static inline void swapMe(KV** a, KV** b) {
+ KV *temp = *a;
+ *a = *b;
+ *b = temp;
+
+ uint16_t a_index = (*a)->array_index;
+ (*a)->array_index = (*b)->array_index;
+ (*b)->array_index = a_index;
}
void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,
const number_t &primary_attr, const number_t &secondary_attr) {
if (size >= MAX_SIZE) {
- if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {
+ if(!is_greater(kvs[0], match_score, primary_attr, secondary_attr)) {
// when incoming value is less than the smallest in the heap, ignore
return;
}
- if(dedup_keys.count(key) != 0) {
- // when the key already exists, ignore
- return ;
+ uint32_t start = 0;
+
+ // When the key already exists and has a greater score, ignore. Otherwise, we have to replace.
+ // NOTE: we don't consider primary and secondary attrs here because they will be the same for a given key.
+ if(keys.count(key) != 0) {
+ const KV* existing = keys.at(key);
+ if(match_score <= existing->match_score) {
+ return ;
+ }
+
+ // replace and sift down
+ start = existing->array_index;
}
- dedup_keys.erase(data[0].key);
- dedup_keys.insert(key);
-
- data[0].key = key;
- data[0].field_id = field_id;
- data[0].query_index = query_index;
- data[0].match_score = match_score;
- data[0].primary_attr = primary_attr;
- data[0].secondary_attr = secondary_attr;
-
- // sift to maintain heap property
- uint32_t i = 0;
- while ((2*i+1) < MAX_SIZE) {
- uint32_t next = (uint32_t) (2 * i + 1);
- if (next+1 < MAX_SIZE && is_greater_kv(data[next], data[next+1])) {
+ keys.erase(kvs[start]->key);
+ keys.emplace(key, kvs[start]);
+
+ kvs[start]->key = key;
+ kvs[start]->field_id = field_id;
+ kvs[start]->query_index = query_index;
+ kvs[start]->array_index = start;
+ kvs[start]->match_score = match_score;
+ kvs[start]->primary_attr = primary_attr;
+ kvs[start]->secondary_attr = secondary_attr;
+
+ // sift down to maintain heap property
+ while ((2*start+1) < MAX_SIZE) {
+ uint32_t next = (uint32_t) (2 * start + 1);
+ if (next+1 < MAX_SIZE && is_greater_kv(kvs[next], kvs[next+1])) {
next++;
}
- if (is_greater_kv(data[i], data[next])) {
- swapMe(data[i], data[next]);
+ if (is_greater_kv(kvs[start], kvs[next])) {
+ swapMe(&kvs[start], &kvs[next]);
} else {
break;
}
- i = next;
+ start = next;
}
} else {
- if(dedup_keys.count(key) != 0) {
- // when the key already exists, ignore
+ uint32_t start = size;
+ bool key_found = false;
+
+ // When the key already exists and has a greater score, ignore. Otherwise, we have to replace
+ if(keys.count(key) != 0) {
+ const KV* existing = keys.at(key);
+ if(match_score <= existing->match_score) {
+ return ;
+ }
+
+ // replace and sift down
+ start = existing->array_index;
+ key_found = true;
+ }
+
+ kvs[start]->key = key;
+ kvs[start]->field_id = field_id;
+ kvs[start]->query_index = query_index;
+ kvs[start]->array_index = start;
+ kvs[start]->match_score = match_score;
+ kvs[start]->primary_attr = primary_attr;
+ kvs[start]->secondary_attr = secondary_attr;
+
+ keys.emplace(key, kvs[start]);
+
+ if(key_found) {
+ // need to sift down if it's a replace
+ while ((2*start+1) < size) {
+ uint32_t next = (uint32_t) (2 * start + 1);
+ if (next+1 < size && is_greater_kv(kvs[next], kvs[next+1])) {
+ next++;
+ }
+
+ if (is_greater_kv(kvs[start], kvs[next])) {
+ swapMe(&kvs[start], &kvs[next]);
+ } else {
+ break;
+ }
+
+ start = next;
+ }
+
return ;
}
- dedup_keys.insert(key);
-
- data[size].key = key;
- data[size].field_id = field_id;
- data[size].query_index = query_index;
- data[size].match_score = match_score;
- data[size].primary_attr = primary_attr;
- data[size].secondary_attr = secondary_attr;
-
- size++;
- for (uint32_t i = size - 1; i > 0;) {
- uint32_t parent = (i-1)/2;
- if (is_greater_kv(data[parent], data[i])) {
- swapMe(data[i], data[parent]);
- i = parent;
+ while(start > 0) {
+ uint32_t parent = (start-1)/2;
+ if (is_greater_kv(kvs[parent], kvs[start])) {
+ swapMe(&kvs[start], &kvs[parent]);
+ start = parent;
} else {
break;
}
}
+
+ if(keys.count(key) != 0) {
+ size++;
+ }
}
}
- static bool is_greater(const struct KV& i, uint64_t match_score, number_t primary_attr, number_t secondary_attr) {
- return std::tie (match_score, primary_attr, secondary_attr) >
- std::tie (i.match_score, i.primary_attr, i.secondary_attr);
+ static bool is_greater(const struct KV* i, uint64_t match_score, number_t primary_attr, number_t secondary_attr) {
+ return std::tie(match_score, primary_attr, secondary_attr) >
+ std::tie(i->match_score, i->primary_attr, i->secondary_attr);
}
- static bool is_greater_kv(const struct KV &i, const struct KV &j) {
- return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.field_id, i.key) >
- std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.field_id, j.key);
+ static bool is_greater_kv(const struct KV* i, const struct KV* j) {
+ return std::tie(i->match_score, i->primary_attr, i->secondary_attr, i->key) >
+ std::tie(j->match_score, j->primary_attr, j->secondary_attr, j->key);
}
// topster must be sorted before iterated upon to remove dead array entries
void sort() {
- std::stable_sort(std::begin(data), std::begin(data) + size, is_greater_kv);
+ std::stable_sort(std::begin(kvs), std::begin(kvs) + size, is_greater_kv);
}
void clear(){
@@ -123,10 +176,10 @@ struct Topster {
}
uint64_t getKeyAt(uint32_t index) {
- return data[index].key;
+ return kvs[index]->key;
}
- KV getKV(uint32_t index) {
- return data[index];
+ KV* getKV(uint32_t index) {
+ return kvs[index];
}
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 9d4f48b3..8f665f86 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -428,7 +428,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// all search queries that were used for generating the results
std::vector<std::vector<art_leaf*>> searched_queries;
- std::vector<Topster<512>::KV> field_order_kvs;
+ std::vector<Topster<512>::KV*> field_order_kvs;
size_t total_found = 0;
// send data to individual index threads
@@ -463,8 +463,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- for(auto & field_order_kv: index->search_params.field_order_kvs) {
- field_order_kv.query_index += searched_queries.size();
+ for(auto* field_order_kv: index->search_params.field_order_kvs) {
+ field_order_kv->query_index += searched_queries.size();
field_order_kvs.push_back(field_order_kv);
}
@@ -515,8 +515,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// construct results array
for(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {
- const auto & field_order_kv = field_order_kvs[field_order_kv_index];
- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);
+ const auto* field_order_kv = field_order_kvs[field_order_kv_index];
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);
std::string json_doc_str;
StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
@@ -536,19 +536,19 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
wrapper_doc["document"] = document;
- //wrapper_doc["match_score"] = field_order_kv.match_score;
- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;
+ //wrapper_doc["match_score"] = field_order_kv->match_score;
+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;
// highlight query words in the result
- const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];
+ const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv->field_id];
field search_field = search_schema.at(field_name);
if(search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
- for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {
+ for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
std::vector<uint16_t> positions;
- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);
+ uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv->key);
uint32_t *indices = new uint32_t[1];
indices[0] = doc_index;
leaf_to_indices.emplace(token_leaf, indices);
@@ -556,7 +556,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// positions in the field of each token in the query
std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
- Index::populate_token_positions(searched_queries[field_order_kv.query_index],
+ Index::populate_token_positions(searched_queries[field_order_kv->query_index],
leaf_to_indices, 0, array_token_positions);
Match match;
@@ -570,8 +570,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- const Match & this_match = Match::match(field_order_kv.key, token_positions);
- uint64_t this_match_score = this_match.get_match_score(1);
+ const Match & this_match = Match::match(field_order_kv->key, token_positions);
+ uint64_t this_match_score = this_match.get_match_score(1, field_order_kv->field_id);
if(this_match_score > match_score) {
match_score = this_match_score;
match = this_match;
diff --git a/src/index.cpp b/src/index.cpp
index 8f5bab2f..83ebd74e 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -600,7 +600,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page, const token_ordering token_order,
const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV> & field_order_kvs,
+ std::vector<Topster<512>::KV*> & field_order_kvs,
size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {
const size_t num_results = (page * per_page);
@@ -639,7 +639,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
// order of fields specified matter: matching docs from earlier fields are more important
for(uint32_t t = 0; t < topster.size && t < num_results; t++) {
- const Topster<512>::KV &kv = topster.getKV(t);
+ Topster<512>::KV* kv = topster.getKV(t);
field_order_kvs.push_back(kv);
}
@@ -890,7 +890,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
char empty_offset_diffs[16];
std::fill_n(empty_offset_diffs, 16, 0);
Match single_token_match = Match(1, 0, 0, empty_offset_diffs);
- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);
for(size_t i=0; i<result_size; i++) {
const uint32_t seq_id = result_ids[i];
@@ -908,7 +908,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
continue;
}
const Match & match = Match::match(seq_id, token_positions);
- uint64_t this_match_score = match.get_match_score(total_cost);
+ uint64_t this_match_score = match.get_match_score(total_cost, field_id);
if(this_match_score > match_score) {
match_score = this_match_score;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index e0387dc0..86d6475b 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -622,7 +622,7 @@ TEST_F(CollectionTest, MultipleFields) {
results = coll_mul_fields->search("thomas", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(4, results["hits"].size());
- ids = {"15", "14", "12", "13"};
+ ids = {"15", "12", "13", "14"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index e0dc5fed..d3d2591a 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -2,7 +2,7 @@
#include "topster.h"
#include "match_score.h"
-TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {
+TEST(TopsterTest, MaxIntValues) {
Topster<5> topster;
struct {
@@ -13,13 +13,13 @@ TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {
int64_t primary_attr;
int64_t secondary_attr;
} data[14] = {
- {1, 0, 1, 10, 20, 30},
- {1, 0, 1, 10, 20, 30},
+ {1, 0, 1, 11, 20, 30},
+ {1, 0, 1, 12, 20, 32},
{1, 0, 2, 4, 20, 30},
{1, 2, 3, 7, 20, 30},
- {1, 0, 4, 11, 20, 30},
- {1, 1, 5, 9, 20, 30},
+ {1, 0, 4, 14, 20, 30},
{1, 1, 5, 9, 20, 30},
+ {1, 1, 5, 10, 20, 32},
{1, 1, 5, 9, 20, 30},
{1, 0, 6, 6, 20, 30},
{1, 2, 7, 6, 22, 30},
@@ -30,45 +30,54 @@ TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {
};
for(int i = 0; i < 14; i++) {
- topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,
+ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, data[i].primary_attr,
data[i].secondary_attr);
}
topster.sort();
- std::vector<uint64_t> ids = {4, 1, 8, 5, 9};
+ std::vector<uint64_t> ids = {4, 1, 5, 8, 9};
for(uint32_t i = 0; i < topster.size; i++) {
EXPECT_EQ(ids[i], topster.getKeyAt(i));
+
+ if(ids[i] == 1) {
+ EXPECT_EQ(12, (int) topster.getKV(i)->match_score);
+ }
+
+ if(ids[i] == 5) {
+ EXPECT_EQ(10, (int) topster.getKV(i)->match_score);
+ }
}
}
-TEST(TopsterTest, StoreMaxFloatValuesWithoutRepetition) {
+TEST(TopsterTest, MaxFloatValues) {
Topster<5> topster;
struct {
+ uint8_t field_id;
uint16_t query_index;
uint64_t key;
uint64_t match_score;
float primary_attr;
int64_t secondary_attr;
} data[12] = {
- {0, 1, 11, 1.09, 30},
- {0, 2, 11, -20, 30},
- {2, 3, 11, -20, 30},
- {0, 4, 11, 7.812, 30},
- {0, 4, 11, 7.812, 30},
- {1, 5, 11, 0.0, 34},
- {0, 6, 11, -22, 30},
- {2, 7, 11, -22, 30},
- {1, 8, 11, -9.998, 30},
- {1, 8, 11, -9.998, 30},
- {0, 9, 11, -9.999, 30},
- {3, 10, 11, -20, 30},
+ {1, 0, 1, 11, 1.09, 30},
+ {1, 0, 2, 11, -20, 30},
+ {1, 2, 3, 11, -20, 30},
+ {1, 0, 4, 11, 7.812, 30},
+ {1, 0, 4, 11, 7.912, 30},
+ {1, 1, 5, 11, 0.0, 34},
+ {1, 0, 6, 11, -22, 30},
+ {1, 2, 7, 11, -22, 30},
+ {1, 1, 8, 11, -9.998, 30},
+ {1, 1, 8, 11, -9.998, 30},
+ {1, 0, 9, 11, -9.999, 30},
+ {1, 3, 10, 11, -20, 30},
};
for(int i = 0; i < 12; i++) {
- topster.add(data[i].key, 1, data[i].query_index, data[i].match_score, data[i].primary_attr,
+ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, data[i].primary_attr,
data[i].secondary_attr);
}