Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Settle for partial matches when the whole query produces no results.
Kishore Nallan4e10fadeb7ed
diff --git a/include/collection.h b/include/collection.hindex 1acf0123..1dec2230 100644--- a/include/collection.h+++ b/include/collection.h@@ -28,7 +28,7 @@ private:static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,long long int n);- void log_leaves(const int max_cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;+ void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,size_t & total_results, const size_t & max_results);diff --git a/src/collection.cpp b/src/collection.cppindex a56f9991..95f7f648 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -90,6 +90,12 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l// every element in `query_suggestion` represents a token and its associated hitsstd::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);+ /*std:: cout << "\nSuggestion: ";+ for(auto suggestion_leaf: query_suggestion) {+ std:: cout << suggestion_leaf->key << " ";+ }+ std::cout << std::endl;*/+// initialize results with the starting element (for further intersection)uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();size_t result_size = query_suggestion[0]->values->ids.getLength();@@ -136,6 +142,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_size_t total_results = 0;std::vector<nlohmann::json> results;Topster<100> topster;+ spp::sparse_hash_map<std::string, uint32_t> token_to_count;auto begin = std::chrono::high_resolution_clock::now();@@ -178,8 +185,9 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1, costs[token_index], 3, leaves);if(!leaves.empty()) {- //log_leaves(max_cost, token, leaves);+ log_leaves(costs[token_index], token, leaves);token_leaves.push_back(leaves);+ token_to_count[token] = leaves.at(0)->values->ids.getLength();} else {// no result when `cost = costs[token_index]` => remove cost for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);@@ -231,8 +239,28 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_n++;}- if(results.size() == 0) {- // FIXME: We could drop certain tokens and try searching again+ if(results.size() == 0 && token_to_count.size() != 0) {+ // Drop certain token with least hits and try searching again+ std::string truncated_query;++ std::vector<std::pair<std::string, uint32_t>> token_count_pairs;+ for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {+ token_count_pairs.push_back(*itr);+ }++ std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]+ (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {+ return a.second > b.second;+ }+ );++ for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {+ if(token_to_count.count(tokens[i]) != 0) {+ truncated_query += " " + token_count_pairs.at(i).first;+ }+ }++ return search(truncated_query, num_typos, num_results);}long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();@@ -241,14 +269,14 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_return results;}-void Collection::log_leaves(const int max_cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {+void Collection::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {+ printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);for(auto i=0; i < leaves.size(); i++) {- printf("%s - ", token.c_str());- printf("%.*s", leaves[i]->key_len, leaves[i]->key);- printf(" - max_cost: %d, - num_ids: %d\n", max_cost, leaves[i]->values->ids.getLength());- for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {+ printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);+ printf("num_ids: %d\n", leaves[i]->values->ids.getLength());+ /*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {printf("id: %d\n", leaves[i]->values->ids.at(j));- }+ }*/}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 3fd48c51..6ee6c12d 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -106,7 +106,16 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {TEST_F(CollectionTest, PartialPhraseSearch) {std::vector<nlohmann::json> results = collection->search("rocket research", 0, 10);- //ASSERT_EQ(1, results.size());+ ASSERT_EQ(4, results.size());++ std::vector<std::string> ids = {"1", "8", "16", "17"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }}TEST_F(CollectionTest, RegressionTest1) {