Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Prefix match must be differentiated from single typo.
Kishore Nallan56247ce6aca2
diff --git a/include/index.h b/include/index.hindex f8d09971..65564625 100644--- a/include/index.h+++ b/include/index.h@@ -29,6 +29,7 @@ struct token_t {struct token_candidates {token_t token;size_t cost;+ bool prefix_search;std::vector<art_leaf*> candidates;};diff --git a/src/art.cpp b/src/art.cppindex f14461b2..b9bd4f31 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1286,7 +1286,7 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last}if(key_len >= term_len) {- // b) we will iterate past term_len to catch trailing typos+ // b) we might iterate past term_len to catch trailing typoscost = cost_row[term_len];if(cost >= min_cost && cost <= max_cost) {return 1;diff --git a/src/index.cpp b/src/index.cppindex 43a0b9e6..c1c65092 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1687,6 +1687,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,drop_tokens_threshold, typo_tokens_threshold);}+ // concat is done only for multi-field searches as `ftopster` will be empty for single-field searchconcat_topster_ids(ftopster, topster_ids);collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);//LOG(INFO) << "topster_ids.size: " << topster_ids.size();@@ -1944,11 +1945,12 @@ void Index::search_field(const uint8_t & field_id,std::vector<art_leaf*> leaves;//LOG(INFO) << "Searching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];+ const bool prefix_search = prefix && (token_index == search_tokens.size()-1);+if(token_cost_cache.count(token_cost_hash) != 0) {leaves = token_cost_cache[token_cost_hash];} else {// prefix should apply only for last token- const bool prefix_search = prefix && (token_index == search_tokens.size()-1);const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;// need less candidates for filtered searches since we already only pick tokens with results@@ -1964,7 +1966,8 @@ void Index::search_field(const uint8_t & field_id,if(!leaves.empty()) {//log_leaves(costs[token_index], token, leaves);- token_candidates_vec.push_back(token_candidates{search_tokens[token_index], costs[token_index], leaves});+ token_candidates_vec.push_back(+ token_candidates{search_tokens[token_index], costs[token_index], prefix_search, leaves});} else {// No result at `cost = costs[token_index]`. Remove `cost` for token and re-do combinationsauto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);@@ -2179,6 +2182,13 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16(uint64_t(100 - total_distance) << 8) |(uint64_t(total_verbatim) << 1));++ /*LOG(INFO) << "Match score: " << match_score << ", for seq_id: " << seq_id+ << " - total_tokens_found: " << total_tokens_found+ << " - total_num_typos: " << total_num_typos+ << " - total_distance: " << total_distance+ << " - total_verbatim: " << total_verbatim+ << " - total_cost: " << total_cost;*/}const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)@@ -2394,13 +2404,19 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &tokeq = ldiv(q.quot, token_candidates_vec[i].candidates.size());actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];- total_cost += token_candidates_vec[i].cost;++ bool exact_match = token_candidates_vec[i].cost == 0 && token_size == actual_query_suggestion[i]->key_len-1;+ bool incr_for_prefix_search = token_candidates_vec[i].prefix_search && !exact_match;++ size_t actual_cost = (2 * token_candidates_vec[i].cost) + uint32_t(incr_for_prefix_search);++ total_cost += actual_cost;token_bits |= 1UL << token_candidates_vec[i].token.position; // sets n-th bit- if(actual_query_suggestion[i]->key_len != token_size+1) {- total_cost++;- }+ /*LOG(INFO) << "suggestion key: " << actual_query_suggestion[i]->key << ", token: "+ << token_candidates_vec[i].token.value << ", actual_cost: " << actual_cost;+ LOG(INFO) << ".";*/}// Sort ascending based on matched documents for each token for faster intersection.diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 5d9ad008..fa086aa2 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -268,6 +268,48 @@ TEST_F(CollectionSpecificTest, PrefixWithTypos) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSpecificTest, PrefixVsExactMatch) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Equivalent Ratios"},+ {"Simplifying Ratios 1"},+ {"Rational and Irrational Numbers"},+ {"Simplifying Ratios 2"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("ration",+ {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {true}).get();++ ASSERT_EQ(4, results["found"].get<size_t>());+ ASSERT_EQ(4, results["hits"].size());++ ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][3]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionSpecificTest, PrefixWithTypos2) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d5a1117e..33e659df 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -415,7 +415,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {ASSERT_EQ(4, results["hits"].size());ASSERT_EQ(13, results["found"].get<uint32_t>());- std::vector<std::string> ids = {"19", "6", "21", "8"};+ std::vector<std::string> ids = {"8", "19", "6", "21"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -429,7 +429,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {ASSERT_EQ(8, results["hits"].size());ASSERT_EQ(8, results["found"].get<uint32_t>());- ids = {"20", "19", "6", "3", "21", "4", "10", "8", "21"};+ ids = {"20", "19", "6", "4", "3", "10", "8", "21"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);