Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Prefix match must be differentiated from single typo.

Kishore Nallan56247ce6aca2
diff --git a/include/index.h b/include/index.h
index f8d09971..65564625 100644
--- a/include/index.h
+++ b/include/index.h
@@ -29,6 +29,7 @@ struct token_t {
struct token_candidates {
token_t token;
size_t cost;
+ bool prefix_search;
std::vector<art_leaf*> candidates;
};
diff --git a/src/art.cpp b/src/art.cpp
index f14461b2..b9bd4f31 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1286,7 +1286,7 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last
}
if(key_len >= term_len) {
- // b) we will iterate past term_len to catch trailing typos
+ // b) we might iterate past term_len to catch trailing typos
cost = cost_row[term_len];
if(cost >= min_cost && cost <= max_cost) {
return 1;
diff --git a/src/index.cpp b/src/index.cpp
index 43a0b9e6..c1c65092 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1687,6 +1687,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
drop_tokens_threshold, typo_tokens_threshold);
}
+ // concat is done only for multi-field searches as `ftopster` will be empty for single-field search
concat_topster_ids(ftopster, topster_ids);
collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);
//LOG(INFO) << "topster_ids.size: " << topster_ids.size();
@@ -1944,11 +1945,12 @@ void Index::search_field(const uint8_t & field_id,
std::vector<art_leaf*> leaves;
//LOG(INFO) << "Searching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];
+ const bool prefix_search = prefix && (token_index == search_tokens.size()-1);
+
if(token_cost_cache.count(token_cost_hash) != 0) {
leaves = token_cost_cache[token_cost_hash];
} else {
// prefix should apply only for last token
- const bool prefix_search = prefix && (token_index == search_tokens.size()-1);
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
// need less candidates for filtered searches since we already only pick tokens with results
@@ -1964,7 +1966,8 @@ void Index::search_field(const uint8_t & field_id,
if(!leaves.empty()) {
//log_leaves(costs[token_index], token, leaves);
- token_candidates_vec.push_back(token_candidates{search_tokens[token_index], costs[token_index], leaves});
+ token_candidates_vec.push_back(
+ token_candidates{search_tokens[token_index], costs[token_index], prefix_search, leaves});
} else {
// No result at `cost = costs[token_index]`. Remove `cost` for token and re-do combinations
auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
@@ -2179,6 +2182,13 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
(uint64_t(100 - total_distance) << 8) |
(uint64_t(total_verbatim) << 1)
);
+
+ /*LOG(INFO) << "Match score: " << match_score << ", for seq_id: " << seq_id
+ << " - total_tokens_found: " << total_tokens_found
+ << " - total_num_typos: " << total_num_typos
+ << " - total_distance: " << total_distance
+ << " - total_verbatim: " << total_verbatim
+ << " - total_cost: " << total_cost;*/
}
const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)
@@ -2394,13 +2404,19 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke
q = ldiv(q.quot, token_candidates_vec[i].candidates.size());
actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];
- total_cost += token_candidates_vec[i].cost;
+
+ bool exact_match = token_candidates_vec[i].cost == 0 && token_size == actual_query_suggestion[i]->key_len-1;
+ bool incr_for_prefix_search = token_candidates_vec[i].prefix_search && !exact_match;
+
+ size_t actual_cost = (2 * token_candidates_vec[i].cost) + uint32_t(incr_for_prefix_search);
+
+ total_cost += actual_cost;
token_bits |= 1UL << token_candidates_vec[i].token.position; // sets n-th bit
- if(actual_query_suggestion[i]->key_len != token_size+1) {
- total_cost++;
- }
+ /*LOG(INFO) << "suggestion key: " << actual_query_suggestion[i]->key << ", token: "
+ << token_candidates_vec[i].token.value << ", actual_cost: " << actual_cost;
+ LOG(INFO) << ".";*/
}
// Sort ascending based on matched documents for each token for faster intersection.
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 5d9ad008..fa086aa2 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -268,6 +268,48 @@ TEST_F(CollectionSpecificTest, PrefixWithTypos) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, PrefixVsExactMatch) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Equivalent Ratios"},
+ {"Simplifying Ratios 1"},
+ {"Rational and Irrational Numbers"},
+ {"Simplifying Ratios 2"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("ration",
+ {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {true}).get();
+
+ ASSERT_EQ(4, results["found"].get<size_t>());
+ ASSERT_EQ(4, results["hits"].size());
+
+ ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][2]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][3]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionSpecificTest, PrefixWithTypos2) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d5a1117e..33e659df 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -415,7 +415,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ(13, results["found"].get<uint32_t>());
- std::vector<std::string> ids = {"19", "6", "21", "8"};
+ std::vector<std::string> ids = {"8", "19", "6", "21"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
@@ -429,7 +429,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
ASSERT_EQ(8, results["hits"].size());
ASSERT_EQ(8, results["found"].get<uint32_t>());
- ids = {"20", "19", "6", "3", "21", "4", "10", "8", "21"};
+ ids = {"20", "19", "6", "4", "3", "10", "8", "21"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);