Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Building up support for prefix based searching and for ranking token suggestions by either frequency or max_score.

Kishore Nallane1526319f757
diff --git a/include/art.h b/include/art.h
index 35f0c0df..f9768bfc 100644
--- a/include/art.h
+++ b/include/art.h
@@ -122,6 +122,11 @@ typedef struct {
uint64_t size;
} art_tree;
+enum token_ordering {
+ FREQUENCY,
+ MAX_SCORE
+};
+
/**
* Initializes an ART tree
* @return 0 on success.
@@ -232,7 +237,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar
* Returns leaves that match a given string within a fuzzy distance of max_cost.
*/
int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,
- const int max_words, std::vector<art_leaf *> &results);
+ const int max_words, const token_ordering token_order, const bool prefix, std::vector<art_leaf *> &results);
static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results);
diff --git a/include/collection.h b/include/collection.h
index 1dec2230..56a17aa8 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -30,7 +30,7 @@ private:
long long int n);
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
- void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
+ void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,
size_t & total_results, const size_t & max_results);
public:
@@ -38,11 +38,12 @@ public:
Collection(std::string state_dir_path);
~Collection();
std::string add(std::string json_str);
- std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results);
+ std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results,
+ const token_ordering token_order = FREQUENCY, const bool prefix = false);
void remove(std::string id);
void score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids,
- size_t result_size) const;
+ const size_t result_size) const;
enum {MAX_SEARCH_TOKENS = 20};
enum {MAX_RESULTS = 100};
diff --git a/src/art.cpp b/src/art.cpp
index 9674634c..0b5064cc 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -30,34 +30,54 @@
static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,
const int term_len, const int* irow, const int* jrow, const int max_cost,
- std::vector<const art_node *> &results);
+ const bool prefix, std::vector<const art_node *> &results);
-/*
- * Comparator for art_leaf struct.
- * We order the leaves descending based on the score.
- */
-bool compare_art_leaf(const art_leaf* a, const art_leaf* b) {
+bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {
return a->values->ids.getLength() > b->values->ids.getLength();
}
-bool compare_art_node(const art_node* a, const art_node* b) {
- uint32_t a_score = 0, b_score = 0;
+bool compare_art_leaf_score(const art_leaf *a, const art_leaf *b) {
+ return a->max_score > b->max_score;
+}
+
+bool compare_art_node_frequency(const art_node *a, const art_node *b) {
+ uint32_t a_value = 0, b_value = 0;
if(IS_LEAF(a)) {
art_leaf* al = (art_leaf *) LEAF_RAW(a);
- a_score = al->values->ids.getLength();
+ a_value = al->values->ids.getLength();
} else {
- a_score = a->max_token_count;
+ a_value = a->max_token_count;
}
if(IS_LEAF(b)) {
art_leaf* bl = (art_leaf *) LEAF_RAW(b);
- b_score = bl->values->ids.getLength();
+ b_value = bl->values->ids.getLength();
} else {
- b_score = b->max_token_count;
+ b_value = b->max_token_count;
}
- return a_score > b_score;
+ return a_value > b_value;
+}
+
+bool compare_art_node_score(const art_node* a, const art_node* b) {
+ uint32_t a_value = 0, b_value = 0;
+
+ if(IS_LEAF(a)) {
+ art_leaf* al = (art_leaf *) LEAF_RAW(a);
+ a_value = al->max_score;
+ } else {
+ a_value = a->max_score;
+ }
+
+ if(IS_LEAF(b)) {
+ art_leaf* bl = (art_leaf *) LEAF_RAW(b);
+ b_value = bl->max_score;
+ } else {
+ b_value = b->max_score;
+ }
+
+ return a_value > b_value;
}
/**
@@ -866,7 +886,7 @@ static uint32_t get_score(art_node* child) {
static int topk_iter(const art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {
printf("INSIDE topk_iter: root->type: %d\n", root->type);
std::priority_queue<art_node *, std::vector<const art_node *>,
- std::function<bool(const art_node*, const art_node*)>> q(compare_art_node);
+ std::function<bool(const art_node*, const art_node*)>> q(compare_art_node_frequency);
q.push(root);
@@ -1135,7 +1155,7 @@ static inline int levenshtein_dist(const int depth, const char p, const char c,
static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,
const int* irow, const int* jrow, const int max_cost,
- std::vector<const art_node *> &results) {
+ const bool prefix, std::vector<const art_node *> &results) {
char child_char;
art_node* child;
@@ -1146,7 +1166,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = ((art_node4*)n)->keys[i];
printf("\n4!child_char: %c, %d, depth: %d", child_char, child_char, depth);
child = ((art_node4*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
}
break;
case NODE16:
@@ -1155,7 +1175,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = ((art_node16*)n)->keys[i];
printf("\n16!child_char: %c, depth: %d", child_char, depth);
child = ((art_node16*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
}
break;
case NODE48:
@@ -1166,7 +1186,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child = ((art_node48*)n)->children[ix - 1];
child_char = (char)i;
printf("\n48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
}
break;
case NODE256:
@@ -1176,7 +1196,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = (char) i;
printf("\n256!child_char: %c, depth: %d", child_char, depth);
child = ((art_node256*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
}
break;
default:
@@ -1195,7 +1215,7 @@ static inline void rotate(int &i, int &j, int &k) {
// e.g. microafot against microsoft
static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,
const int term_len, const int* irow, const int* jrow, const int max_cost,
- std::vector<const art_node *> &results) {
+ const bool prefix, std::vector<const art_node *> &results) {
const int columns = term_len+1;
int i=0, j=1, k=2;
int row0[columns];
@@ -1261,14 +1281,21 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons
depth += n->partial_len;
printf("\ncost: %d", cost);
- art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, results);
+ // For a prefix search, we store the node and not recurse further right now
+ if(prefix && depth >= term_len-1 && rows[j][columns-1] <= max_cost) {
+ results.push_back(n);
+ return ;
+ }
+
+ art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, prefix, results);
}
/**
* Returns leaves that match a given string within a fuzzy distance of max_cost.
*/
-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
- const int max_cost, const int max_words, std::vector<art_leaf *> &results) {
+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,
+ const int max_words, const token_ordering token_order, const bool prefix,
+ std::vector<art_leaf *> &results) {
std::vector<const art_node*> nodes;
int irow[term_len + 1];
@@ -1282,12 +1309,17 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
if(IS_LEAF(t->root)) {
art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
- art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, nodes);
+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);
} else {
- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, nodes);
+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);
+ }
+
+ if(token_order == FREQUENCY) {
+ std::sort(nodes.begin(), nodes.end(), compare_art_node_frequency);
+ } else {
+ std::sort(nodes.begin(), nodes.end(), compare_art_node_score);
}
- std::sort(nodes.begin(), nodes.end(), compare_art_node);
long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;
@@ -1297,7 +1329,12 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
topk_iter(node, term_len, max_words, results);
}
- std::sort(results.begin(), results.end(), compare_art_leaf);
+ if(token_order == FREQUENCY) {
+ std::sort(results.begin(), results.end(), compare_art_leaf_frequency);
+ } else {
+ std::sort(results.begin(), results.end(), compare_art_leaf_score);
+ }
+
time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
std::cout << "Time taken for topk_iter: " << time_micro << "us" << std::endl;
return 0;
diff --git a/src/collection.cpp b/src/collection.cpp
index 3d4a3422..ecc746ea 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -80,8 +80,9 @@ std::string Collection::add(std::string json_str) {
return document["id"];
}
-void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
- size_t & total_results, const size_t & max_results) {
+void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,
+ std::vector<nlohmann::json> & results, size_t & total_results,
+ const size_t & max_results) {
const size_t combination_limit = 10;
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
@@ -113,14 +114,22 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l
}
// go through each matching document id and calculate match score
+ Topster<100> topster;
score_results(topster, query_suggestion, result_ids, result_size);
-
total_results += result_size;
delete[] result_ids;
+ topster.sort();
+
+ for (uint32_t i = 0; i < topster.size; i++) {
+ uint64_t seq_id = topster.getKeyAt(i);
+ std::string value;
+ store->get(get_seq_id_key((uint32_t) seq_id), value);
+ nlohmann::json document = nlohmann::json::parse(value);
+ results.push_back(document);
+ }
if(total_results >= max_results) break;
}
-
}
/*
@@ -132,7 +141,8 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-std::vector<nlohmann::json> Collection::search(std::string query, const int num_typos, const size_t num_results) {
+std::vector<nlohmann::json> Collection::search(std::string query, const int num_typos, const size_t num_results,
+ const token_ordering token_order, const bool prefix) {
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::string> tokens;
@@ -143,10 +153,13 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
size_t total_results = 0;
std::vector<nlohmann::json> results;
- Topster<100> topster;
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cache;
+
+ // Used to drop the least occurring token(s) for partial searches
+ spp::sparse_hash_map<std::string, uint32_t> token_to_count;
+
std::vector<std::vector<int>> token_to_costs;
std::vector<int> all_costs;
@@ -182,21 +195,24 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
while(token_index < tokens.size()) {
// For each token, look up the generated cost for this iteration and search using that cost
std::string token = tokens[token_index];
+ const std::string token_cost_hash = token + std::to_string(costs[token_index]);
+
std::vector<art_leaf*> leaves;
- if(token_cache.count(token) != 0) {
- leaves = token_cache[token];
+ if(token_cache.count(token_cost_hash) != 0) {
+ leaves = token_cache[token_cost_hash];
} else {
art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1,
- costs[token_index], 3, leaves);
+ costs[token_index], 3, token_order, prefix, leaves);
if(!leaves.empty()) {
- token_cache.emplace(token, leaves);
+ token_cache.emplace(token_cost_hash, leaves);
}
}
if(!leaves.empty()) {
log_leaves(costs[token_index], token, leaves);
token_leaves.push_back(leaves);
+ token_to_count[token] = leaves.at(0)->values->ids.getLength();
} else {
// No result at `cost = costs[token_index]` => remove cost for token and re-do combinations
auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
@@ -228,16 +244,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
if(token_leaves.size() != 0 && !retry_with_larger_cost) {
// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,
// go ahead and search for candidates with what we have so far
- search_candidates(token_leaves, topster, total_results, max_results);
- topster.sort();
-
- for (uint32_t i = 0; i < topster.size; i++) {
- uint64_t seq_id = topster.getKeyAt(i);
- std::string value;
- store->get(get_seq_id_key((uint32_t) seq_id), value);
- nlohmann::json document = nlohmann::json::parse(value);
- results.push_back(document);
- }
+ search_candidates(token_leaves, results, total_results, max_results);
if (total_results > 0) {
// Unless there are results, we continue outerloop (looking at tokens with greater cost)
@@ -248,23 +255,23 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
n++;
}
- if(results.size() == 0 && token_cache.size() != 0) {
+ if(results.size() == 0 && token_to_count.size() != 0) {
// Drop certain token with least hits and try searching again
std::string truncated_query;
std::vector<std::pair<std::string, uint32_t>> token_count_pairs;
- for (auto itr = token_cache.begin(); itr != token_cache.end(); ++itr) {
- token_count_pairs.push_back(std::make_pair(itr->first, itr->second.at(0)->values->ids.getLength()));
+ for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {
+ token_count_pairs.push_back(*itr);
}
std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]
- (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {
- return a.second > b.second;
- }
+ (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {
+ return a.second > b.second;
+ }
);
for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {
- if(token_cache.count(tokens[i]) != 0) {
+ if(token_to_count.count(tokens[i]) != 0) {
truncated_query += " " + token_count_pairs.at(i).first;
}
}
@@ -282,7 +289,7 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:
printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
for(auto i=0; i < leaves.size(); i++) {
printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
- printf("num_ids: %d\n", leaves[i]->values->ids.getLength());
+ printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);
/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
printf("id: %d\n", leaves[i]->values->ids.at(j));
}*/
@@ -290,7 +297,7 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:
}
void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,
- const uint32_t *result_ids, size_t result_size) const {
+ const uint32_t *result_ids, const size_t result_size) const {
for(auto i=0; i<result_size; i++) {
uint32_t doc_id = result_ids[i];
std::vector<std::vector<uint16_t>> token_positions;
diff --git a/src/main/server.cpp b/src/main/server.cpp
index 1791d381..c1b6b197 100644
--- a/src/main/server.cpp
+++ b/src/main/server.cpp
@@ -70,7 +70,7 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {
printf("Query: %s\n", query_map["q"].c_str());
auto begin = std::chrono::high_resolution_clock::now();
- std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100);
+ std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, FREQUENCY, false);
nlohmann::json json_array = nlohmann::json::array();
for(nlohmann::json& result: results) {
json_array.push_back(result);
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 6c5fca51..b2a052ac 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -586,22 +586,22 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {
EXPECT_EQ(1, l->values->ids.at(0));
std::vector<art_leaf*> leaves;
- art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
const char* implement_key_typo1 = "implment";
const char* implement_key_typo2 = "implwnent";
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, FREQUENCY, false, leaves);
ASSERT_EQ(0, leaves.size());
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
}
@@ -631,43 +631,43 @@ TEST(ArtTest, test_art_fuzzy_search) {
std::vector<art_leaf*> leaves;
// transpose
- art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);
// transpose + missing
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);
// missing char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("gaberlunzie", (const char *)leaves.at(0)->key);
// extra char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("highliving", (const char *)leaves.at(0)->key);
// substituted char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("racemiferous", (const char *)leaves.at(0)->key);
// missing char + extra char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("Saarbrucken", (const char *)leaves.at(0)->key);
// multiple matching results
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(10, leaves.size());
std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index ee9b6b77..81c4c65e 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -129,6 +129,69 @@ TEST_F(CollectionTest, QueryWithTypo) {
results = collection->search("fer thx", 1, 10);
std::vector<std::string> ids = {"1", "10", "13"};
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+}
+
+TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
+ std::vector<nlohmann::json> results = collection->search("loox", 1, 2, MAX_SCORE, false);
+ ASSERT_EQ(2, results.size());
+ std::vector<std::string> ids = {"22", "23"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ results = collection->search("loox", 1, 3, FREQUENCY, false);
+ ASSERT_EQ(3, results.size());
+ ids = {"3", "12", "24"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // Check total ordering
+
+ results = collection->search("loox", 1, 10, FREQUENCY, false);
+ ASSERT_EQ(5, results.size());
+ ids = {"3", "12", "24", "22", "23"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ results = collection->search("loox", 1, 10, MAX_SCORE, false);
+ ASSERT_EQ(5, results.size());
+ ids = {"22", "23", "3", "12", "24"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+}
+
+TEST_F(CollectionTest, TextContainingAnActualTypo) {
+ // A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens
+ std::vector<nlohmann::json> results = collection->search("ISX what", 1, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results.size());
+
+ std::vector<std::string> ids = {"19", "6", "21", "8"};
+
for(size_t i = 0; i < results.size(); i++) {
nlohmann::json result = results.at(i);
std::string result_id = result["id"];
diff --git a/test/documents.jsonl b/test/documents.jsonl
index 00e4aed7..130fc0a0 100644
--- a/test/documents.jsonl
+++ b/test/documents.jsonl
@@ -17,5 +17,8 @@
{"points":8,"title":"How much does it cost to launch (right from start) a rocket today?"}
{"points":16,"title":"Difference between Space Dynamics & Astrodynamics in engineering perspective?"}
{"points":18,"title":"What kind of biological research does ISS do?"}
-{"points":10,"title":"What kinds of radiation hit ISS?"}
-{"points":7,"title":"What kinds of things have been tossed out of ISS?"}
\ No newline at end of file
+{"points":10,"title":"Which kinds of radiation hit ISX ?"}
+{"points":7,"title":"What kinds of things have been tossed out of ISS in space?"}
+{"points":17,"title":"What does triple redundant closed loop digital avionics system mean?"}
+{"points":11,"title":"How are rockets guided to follow specific loop trajectory?"}
+{"points":8,"title":"What do remotely controlled bolts look like?"}
\ No newline at end of file