Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Only last token should prefix search during drop tokens.

Kishore Nallanf45416be0c20
diff --git a/include/index.h b/include/index.h
index 1309abea..effe2bc8 100644
--- a/include/index.h
+++ b/include/index.h
@@ -29,6 +29,11 @@ using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;
struct token_t {
size_t position;
std::string value;
+ bool prefix;
+
+ token_t(size_t position, const std::string& value, bool prefix): position(position), value(value), prefix(prefix) {
+
+ }
};
struct token_candidates {
diff --git a/src/index.cpp b/src/index.cpp
index 03faac11..19885cba 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1923,7 +1923,8 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
std::vector<token_t> window_tokens;
std::set<std::string> window_tokens_set;
for (size_t i = start_index; i < start_index + window_len; i++) {
- window_tokens.push_back({i, tokens[i]});
+ bool is_prefix = (i == (start_index + window_len - 1));
+ window_tokens.emplace_back(i, tokens[i], is_prefix);
window_tokens_set.emplace(tokens[i]);
}
@@ -2100,14 +2101,16 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
for(size_t i = 0; i < num_search_fields; i++) {
std::vector<token_t> q_include_pos_tokens;
for(size_t j=0; j < field_query_tokens[i].q_include_tokens.size(); j++) {
- q_include_pos_tokens.push_back({j, field_query_tokens[i].q_include_tokens[j]});
+ bool is_prefix = (j == field_query_tokens[i].q_include_tokens.size()-1);
+ q_include_pos_tokens.emplace_back(j, field_query_tokens[i].q_include_tokens[j], is_prefix);
}
std::vector<std::vector<token_t>> q_pos_synonyms;
for(const auto& q_syn_vec: field_query_tokens[i].q_synonyms) {
std::vector<token_t> q_pos_syn;
for(size_t j=0; j < q_syn_vec.size(); j++) {
- q_pos_syn.push_back({j, q_syn_vec[j]});
+ bool is_prefix = (j == q_syn_vec.size()-1);
+ q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix);
}
q_pos_synonyms.emplace_back(q_pos_syn);
}
@@ -2543,8 +2546,9 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
std::vector<token_t> search_tokens, qtokens;
for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {
- search_tokens.emplace_back(token_t{qtoken_index, query_tokens[qtoken_index]});
- qtokens.emplace_back(token_t{qtoken_index, query_tokens[qtoken_index]});
+ bool is_prefix = (qtoken_index == query_tokens.size()-1);
+ search_tokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix);
+ qtokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix);
}
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -2893,14 +2897,14 @@ void Index::search_field(const uint8_t & field_id,
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
- //LOG(INFO) << "Searching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];
+ const bool prefix_search = prefix && search_tokens[token_index].prefix;
- const bool prefix_search = prefix && (token_index == search_tokens.size()-1);
+ /*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
+ << token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
if(token_cost_cache.count(token_cost_hash) != 0) {
leaves = token_cost_cache[token_cost_hash];
} else {
- // prefix should apply only for last token
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
//auto begin = std::chrono::high_resolution_clock::now();
@@ -3008,13 +3012,13 @@ void Index::search_field(const uint8_t & field_id,
// drop from right
size_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;
for(size_t i=0; i <= end_index; i++) {
- truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});
+ truncated_tokens.emplace_back(query_tokens[i].position, query_tokens[i].value, query_tokens[i].prefix);
}
} else {
// drop from left
size_t start_index = (num_tokens_dropped - mid_index);
for(size_t i=start_index; i<query_tokens.size(); i++) {
- truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});
+ truncated_tokens.emplace_back(query_tokens[i].position, query_tokens[i].value, query_tokens[i].prefix);
}
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 8ae33b22..c148b659 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1711,8 +1711,6 @@ TEST_F(CollectionSpecificTest, RepeatingStringArrayTokens) {
}
TEST_F(CollectionSpecificTest, HighlightOnPrefixRegression) {
- std::vector<std::string> tags;
-
// when the first document containing a token already cannot fit compact posting list
std::vector<field> fields = {field("title", field_types::STRING, false),};
@@ -1727,8 +1725,46 @@ TEST_F(CollectionSpecificTest, HighlightOnPrefixRegression) {
auto results = coll1->search("and", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
- LOG(INFO) << results;
-
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Dog Shoemaker";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["title"] = "Shoe and Sock";
+ doc2["points"] = 200;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("shoe cat", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ results = coll1->search("cat shoe", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+}