Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Building up support for prefix based searching and for ranking token suggestions by either frequency or max_score.
Kishore Nallane1526319f757
diff --git a/include/art.h b/include/art.hindex 35f0c0df..f9768bfc 100644--- a/include/art.h+++ b/include/art.h@@ -122,6 +122,11 @@ typedef struct {uint64_t size;} art_tree;+enum token_ordering {+ FREQUENCY,+ MAX_SCORE+};+/*** Initializes an ART tree* @return 0 on success.@@ -232,7 +237,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar* Returns leaves that match a given string within a fuzzy distance of max_cost.*/int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,- const int max_words, std::vector<art_leaf *> &results);+ const int max_words, const token_ordering token_order, const bool prefix, std::vector<art_leaf *> &results);static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results);diff --git a/include/collection.h b/include/collection.hindex 1dec2230..56a17aa8 100644--- a/include/collection.h+++ b/include/collection.h@@ -30,7 +30,7 @@ private:long long int n);void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;- void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,+ void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,size_t & total_results, const size_t & max_results);public:@@ -38,11 +38,12 @@ public:Collection(std::string state_dir_path);~Collection();std::string add(std::string json_str);- std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results);+ std::vector<nlohmann::json> search(std::string query, const int num_typos, const size_t num_results,+ const token_ordering token_order = FREQUENCY, const bool prefix = false);void remove(std::string id);void score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids,- size_t result_size) const;+ const size_t result_size) const;enum {MAX_SEARCH_TOKENS = 20};enum {MAX_RESULTS = 100};diff --git a/src/art.cpp b/src/art.cppindex 9674634c..0b5064cc 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -30,34 +30,54 @@static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,const int term_len, const int* irow, const int* jrow, const int max_cost,- std::vector<const art_node *> &results);+ const bool prefix, std::vector<const art_node *> &results);-/*- * Comparator for art_leaf struct.- * We order the leaves descending based on the score.- */-bool compare_art_leaf(const art_leaf* a, const art_leaf* b) {+bool compare_art_leaf_frequency(const art_leaf *a, const art_leaf *b) {return a->values->ids.getLength() > b->values->ids.getLength();}-bool compare_art_node(const art_node* a, const art_node* b) {- uint32_t a_score = 0, b_score = 0;+bool compare_art_leaf_score(const art_leaf *a, const art_leaf *b) {+ return a->max_score > b->max_score;+}++bool compare_art_node_frequency(const art_node *a, const art_node *b) {+ uint32_t a_value = 0, b_value = 0;if(IS_LEAF(a)) {art_leaf* al = (art_leaf *) LEAF_RAW(a);- a_score = al->values->ids.getLength();+ a_value = al->values->ids.getLength();} else {- a_score = a->max_token_count;+ a_value = a->max_token_count;}if(IS_LEAF(b)) {art_leaf* bl = (art_leaf *) LEAF_RAW(b);- b_score = bl->values->ids.getLength();+ b_value = bl->values->ids.getLength();} else {- b_score = b->max_token_count;+ b_value = b->max_token_count;}- return a_score > b_score;+ return a_value > b_value;+}++bool compare_art_node_score(const art_node* a, const art_node* b) {+ uint32_t a_value = 0, b_value = 0;++ if(IS_LEAF(a)) {+ art_leaf* al = (art_leaf *) LEAF_RAW(a);+ a_value = al->max_score;+ } else {+ a_value = a->max_score;+ }++ if(IS_LEAF(b)) {+ art_leaf* bl = (art_leaf *) LEAF_RAW(b);+ b_value = bl->max_score;+ } else {+ b_value = b->max_score;+ }++ return a_value > b_value;}/**@@ -866,7 +886,7 @@ static uint32_t get_score(art_node* child) {static int topk_iter(const art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {printf("INSIDE topk_iter: root->type: %d\n", root->type);std::priority_queue<art_node *, std::vector<const art_node *>,- std::function<bool(const art_node*, const art_node*)>> q(compare_art_node);+ std::function<bool(const art_node*, const art_node*)>> q(compare_art_node_frequency);q.push(root);@@ -1135,7 +1155,7 @@ static inline int levenshtein_dist(const int depth, const char p, const char c,static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,const int* irow, const int* jrow, const int max_cost,- std::vector<const art_node *> &results) {+ const bool prefix, std::vector<const art_node *> &results) {char child_char;art_node* child;@@ -1146,7 +1166,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = ((art_node4*)n)->keys[i];printf("\n4!child_char: %c, %d, depth: %d", child_char, child_char, depth);child = ((art_node4*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);}break;case NODE16:@@ -1155,7 +1175,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = ((art_node16*)n)->keys[i];printf("\n16!child_char: %c, depth: %d", child_char, depth);child = ((art_node16*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);}break;case NODE48:@@ -1166,7 +1186,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild = ((art_node48*)n)->children[ix - 1];child_char = (char)i;printf("\n48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);}break;case NODE256:@@ -1176,7 +1196,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = (char) i;printf("\n256!child_char: %c, depth: %d", child_char, depth);child = ((art_node256*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);}break;default:@@ -1195,7 +1215,7 @@ static inline void rotate(int &i, int &j, int &k) {// e.g. microafot against microsoftstatic void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,const int term_len, const int* irow, const int* jrow, const int max_cost,- std::vector<const art_node *> &results) {+ const bool prefix, std::vector<const art_node *> &results) {const int columns = term_len+1;int i=0, j=1, k=2;int row0[columns];@@ -1261,14 +1281,21 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, consdepth += n->partial_len;printf("\ncost: %d", cost);- art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, results);+ // For a prefix search, we store the node and not recurse further right now+ if(prefix && depth >= term_len-1 && rows[j][columns-1] <= max_cost) {+ results.push_back(n);+ return ;+ }++ art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, prefix, results);}/*** Returns leaves that match a given string within a fuzzy distance of max_cost.*/-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,- const int max_cost, const int max_words, std::vector<art_leaf *> &results) {+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,+ const int max_words, const token_ordering token_order, const bool prefix,+ std::vector<art_leaf *> &results) {std::vector<const art_node*> nodes;int irow[term_len + 1];@@ -1282,12 +1309,17 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,if(IS_LEAF(t->root)) {art_leaf *l = (art_leaf *) LEAF_RAW(t->root);- art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, nodes);+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);} else {- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, nodes);+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);+ }++ if(token_order == FREQUENCY) {+ std::sort(nodes.begin(), nodes.end(), compare_art_node_frequency);+ } else {+ std::sort(nodes.begin(), nodes.end(), compare_art_node_score);}- std::sort(nodes.begin(), nodes.end(), compare_art_node);long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;@@ -1297,7 +1329,12 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,topk_iter(node, term_len, max_words, results);}- std::sort(results.begin(), results.end(), compare_art_leaf);+ if(token_order == FREQUENCY) {+ std::sort(results.begin(), results.end(), compare_art_leaf_frequency);+ } else {+ std::sort(results.begin(), results.end(), compare_art_leaf_score);+ }+time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();std::cout << "Time taken for topk_iter: " << time_micro << "us" << std::endl;return 0;diff --git a/src/collection.cpp b/src/collection.cppindex 3d4a3422..ecc746ea 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -80,8 +80,9 @@ std::string Collection::add(std::string json_str) {return document["id"];}-void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,- size_t & total_results, const size_t & max_results) {+void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,+ std::vector<nlohmann::json> & results, size_t & total_results,+ const size_t & max_results) {const size_t combination_limit = 10;auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);@@ -113,14 +114,22 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l}// go through each matching document id and calculate match score+ Topster<100> topster;score_results(topster, query_suggestion, result_ids, result_size);-total_results += result_size;delete[] result_ids;+ topster.sort();++ for (uint32_t i = 0; i < topster.size; i++) {+ uint64_t seq_id = topster.getKeyAt(i);+ std::string value;+ store->get(get_seq_id_key((uint32_t) seq_id), value);+ nlohmann::json document = nlohmann::json::parse(value);+ results.push_back(document);+ }if(total_results >= max_results) break;}-}/*@@ -132,7 +141,8 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-std::vector<nlohmann::json> Collection::search(std::string query, const int num_typos, const size_t num_results) {+std::vector<nlohmann::json> Collection::search(std::string query, const int num_typos, const size_t num_results,+ const token_ordering token_order, const bool prefix) {auto begin = std::chrono::high_resolution_clock::now();std::vector<std::string> tokens;@@ -143,10 +153,13 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_size_t total_results = 0;std::vector<nlohmann::json> results;- Topster<100> topster;// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cache;++ // Used to drop the least occurring token(s) for partial searches+ spp::sparse_hash_map<std::string, uint32_t> token_to_count;+std::vector<std::vector<int>> token_to_costs;std::vector<int> all_costs;@@ -182,21 +195,24 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_while(token_index < tokens.size()) {// For each token, look up the generated cost for this iteration and search using that coststd::string token = tokens[token_index];+ const std::string token_cost_hash = token + std::to_string(costs[token_index]);+std::vector<art_leaf*> leaves;- if(token_cache.count(token) != 0) {- leaves = token_cache[token];+ if(token_cache.count(token_cost_hash) != 0) {+ leaves = token_cache[token_cost_hash];} else {art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1,- costs[token_index], 3, leaves);+ costs[token_index], 3, token_order, prefix, leaves);if(!leaves.empty()) {- token_cache.emplace(token, leaves);+ token_cache.emplace(token_cost_hash, leaves);}}if(!leaves.empty()) {log_leaves(costs[token_index], token, leaves);token_leaves.push_back(leaves);+ token_to_count[token] = leaves.at(0)->values->ids.getLength();} else {// No result at `cost = costs[token_index]` => remove cost for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);@@ -228,16 +244,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_if(token_leaves.size() != 0 && !retry_with_larger_cost) {// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,// go ahead and search for candidates with what we have so far- search_candidates(token_leaves, topster, total_results, max_results);- topster.sort();-- for (uint32_t i = 0; i < topster.size; i++) {- uint64_t seq_id = topster.getKeyAt(i);- std::string value;- store->get(get_seq_id_key((uint32_t) seq_id), value);- nlohmann::json document = nlohmann::json::parse(value);- results.push_back(document);- }+ search_candidates(token_leaves, results, total_results, max_results);if (total_results > 0) {// Unless there are results, we continue outerloop (looking at tokens with greater cost)@@ -248,23 +255,23 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_n++;}- if(results.size() == 0 && token_cache.size() != 0) {+ if(results.size() == 0 && token_to_count.size() != 0) {// Drop certain token with least hits and try searching againstd::string truncated_query;std::vector<std::pair<std::string, uint32_t>> token_count_pairs;- for (auto itr = token_cache.begin(); itr != token_cache.end(); ++itr) {- token_count_pairs.push_back(std::make_pair(itr->first, itr->second.at(0)->values->ids.getLength()));+ for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {+ token_count_pairs.push_back(*itr);}std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]- (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {- return a.second > b.second;- }+ (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {+ return a.second > b.second;+ });for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {- if(token_cache.count(tokens[i]) != 0) {+ if(token_to_count.count(tokens[i]) != 0) {truncated_query += " " + token_count_pairs.at(i).first;}}@@ -282,7 +289,7 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);for(auto i=0; i < leaves.size(); i++) {printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);- printf("num_ids: %d\n", leaves[i]->values->ids.getLength());+ printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {printf("id: %d\n", leaves[i]->values->ids.at(j));}*/@@ -290,7 +297,7 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:}void Collection::score_results(Topster<100> &topster, const std::vector<art_leaf *> &query_suggestion,- const uint32_t *result_ids, size_t result_size) const {+ const uint32_t *result_ids, const size_t result_size) const {for(auto i=0; i<result_size; i++) {uint32_t doc_id = result_ids[i];std::vector<std::vector<uint16_t>> token_positions;diff --git a/src/main/server.cpp b/src/main/server.cppindex 1791d381..c1b6b197 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -70,7 +70,7 @@ static int get_search(h2o_handler_t *self, h2o_req_t *req) {printf("Query: %s\n", query_map["q"].c_str());auto begin = std::chrono::high_resolution_clock::now();- std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100);+ std::vector<nlohmann::json> results = collection->search(query_map["q"], std::stoi(query_map[NUM_TYPOS]), 100, FREQUENCY, false);nlohmann::json json_array = nlohmann::json::array();for(nlohmann::json& result: results) {json_array.push_back(result);diff --git a/test/art_test.cpp b/test/art_test.cppindex 6c5fca51..b2a052ac 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -586,22 +586,22 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {EXPECT_EQ(1, l->values->ids.at(0));std::vector<art_leaf*> leaves;- art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());const char* implement_key_typo1 = "implment";const char* implement_key_typo2 = "implwnent";leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, FREQUENCY, false, leaves);ASSERT_EQ(0, leaves.size());leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());}@@ -631,43 +631,43 @@ TEST(ArtTest, test_art_fuzzy_search) {std::vector<art_leaf*> leaves;// transpose- art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);// transpose + missingleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);// missing charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("gaberlunzie", (const char *)leaves.at(0)->key);// extra charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("highliving", (const char *)leaves.at(0)->key);// substituted charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("racemiferous", (const char *)leaves.at(0)->key);// missing char + extra charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("Saarbrucken", (const char *)leaves.at(0)->key);// multiple matching resultsleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(10, leaves.size());std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};diff --git a/test/collection_test.cpp b/test/collection_test.cppindex ee9b6b77..81c4c65e 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -129,6 +129,69 @@ TEST_F(CollectionTest, QueryWithTypo) {results = collection->search("fer thx", 1, 10);std::vector<std::string> ids = {"1", "10", "13"};+ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}++TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {+ std::vector<nlohmann::json> results = collection->search("loox", 1, 2, MAX_SCORE, false);+ ASSERT_EQ(2, results.size());+ std::vector<std::string> ids = {"22", "23"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ results = collection->search("loox", 1, 3, FREQUENCY, false);+ ASSERT_EQ(3, results.size());+ ids = {"3", "12", "24"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // Check total ordering++ results = collection->search("loox", 1, 10, FREQUENCY, false);+ ASSERT_EQ(5, results.size());+ ids = {"3", "12", "24", "22", "23"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ results = collection->search("loox", 1, 10, MAX_SCORE, false);+ ASSERT_EQ(5, results.size());+ ids = {"22", "23", "3", "12", "24"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}++TEST_F(CollectionTest, TextContainingAnActualTypo) {+ // A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens+ std::vector<nlohmann::json> results = collection->search("ISX what", 1, 10, FREQUENCY, false);+ ASSERT_EQ(4, results.size());++ std::vector<std::string> ids = {"19", "6", "21", "8"};+for(size_t i = 0; i < results.size(); i++) {nlohmann::json result = results.at(i);std::string result_id = result["id"];diff --git a/test/documents.jsonl b/test/documents.jsonlindex 00e4aed7..130fc0a0 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -17,5 +17,8 @@{"points":8,"title":"How much does it cost to launch (right from start) a rocket today?"}{"points":16,"title":"Difference between Space Dynamics & Astrodynamics in engineering perspective?"}{"points":18,"title":"What kind of biological research does ISS do?"}-{"points":10,"title":"What kinds of radiation hit ISS?"}-{"points":7,"title":"What kinds of things have been tossed out of ISS?"}\ No newline at end of file+{"points":10,"title":"Which kinds of radiation hit ISX ?"}+{"points":7,"title":"What kinds of things have been tossed out of ISS in space?"}+{"points":17,"title":"What does triple redundant closed loop digital avionics system mean?"}+{"points":11,"title":"How are rockets guided to follow specific loop trajectory?"}+{"points":8,"title":"What do remotely controlled bolts look like?"}\ No newline at end of file