Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix prefix based searches.
Kishore Nallan17a7297384e1
diff --git a/src/art.cpp b/src/art.cppindex 0b5064cc..2b652fe4 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -883,27 +883,21 @@ static uint32_t get_score(art_node* child) {return child->max_token_count;}-static int topk_iter(const art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {+static int topk_iter(const art_node *root, int max_results, std::vector<art_leaf*> & results) {printf("INSIDE topk_iter: root->type: %d\n", root->type);std::priority_queue<art_node *, std::vector<const art_node *>,std::function<bool(const art_node*, const art_node*)>> q(compare_art_node_frequency);q.push(root);- while(!q.empty() && results.size() < k) {+ while(!q.empty() && results.size() < max_results) {art_node *n = (art_node *) q.top();q.pop();if (!n) continue;if (IS_LEAF(n)) {art_leaf *l = (art_leaf *) LEAF_RAW(n);- //printf("\nTOPK LEAF: %.*s", l->key_len, l->key);- //std::cout << ", SCORE: " << l->token_count << std::endl;- // This basically ignores tokens whose prefix match perfectly, but exceeding overall threshold- int diff = term_len - (int) l->key_len;- if(diff >= -2 && diff <= 2) {- results.push_back(l);- }+ results.push_back(l);continue;}@@ -1326,7 +1320,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,begin = std::chrono::high_resolution_clock::now();for(auto node: nodes) {- topk_iter(node, term_len, max_words, results);+ topk_iter(node, max_words, results);}if(token_order == FREQUENCY) {diff --git a/src/collection.cpp b/src/collection.cppindex ecc746ea..a3508ab1 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -87,8 +87,11 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_lauto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);+ // For deduplication. If 2 query suggestions give a document as result, ensure that only one is returned+ spp::sparse_hash_set<uint64_t> dedup_seq_ids;+for(long long n=0; n<N && n<combination_limit; ++n) {- // every element in `query_suggestion` represents a token and its associated hits+ // every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);/*std:: cout << "\nSuggestion: ";@@ -122,10 +125,13 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_lfor (uint32_t i = 0; i < topster.size; i++) {uint64_t seq_id = topster.getKeyAt(i);- std::string value;- store->get(get_seq_id_key((uint32_t) seq_id), value);- nlohmann::json document = nlohmann::json::parse(value);- results.push_back(document);+ if(dedup_seq_ids.count(seq_id) == 0) {+ std::string value;+ store->get(get_seq_id_key((uint32_t) seq_id), value);+ nlohmann::json document = nlohmann::json::parse(value);+ results.push_back(document);+ dedup_seq_ids.emplace(seq_id);+ }}if(total_results >= max_results) break;@@ -202,7 +208,8 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_if(token_cache.count(token_cost_hash) != 0) {leaves = token_cache[token_cost_hash];} else {- art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1,+ int token_len = prefix ? (int) token.length() : (int) token.length() + 1;+ art_fuzzy_search(&t, (const unsigned char *) token.c_str(), token_len,costs[token_index], 3, token_order, prefix, leaves);if(!leaves.empty()) {token_cache.emplace(token_cost_hash, leaves);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 81c4c65e..a6d9f351 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -192,6 +192,30 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {std::vector<std::string> ids = {"19", "6", "21", "8"};+ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}++TEST_F(CollectionTest, PrefixSearching) {+ std::vector<nlohmann::json> results = collection->search("ex", 0, 10, FREQUENCY, true);+ ASSERT_EQ(2, results.size());+ std::vector<std::string> ids = {"12", "6"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ results = collection->search("ex", 0, 10, MAX_SCORE, true);+ ASSERT_EQ(2, results.size());+ ids = {"6", "12"};+for(size_t i = 0; i < results.size(); i++) {nlohmann::json result = results.at(i);std::string result_id = result["id"];