Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support prefix matching of filter value. (#1597)

Harpreet Sangardaad8df50beb
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 033ed005..5c904d4b 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -1144,7 +1144,13 @@ void filter_result_iterator_t::init() {
} else if (f.is_string()) {
art_tree* t = index->search_index.at(a_filter.field_name);
- for (const std::string& filter_value : a_filter.values) {
+ for (std::string filter_value : a_filter.values) {
+ auto is_prefix_match = filter_value.size() > 2 && filter_value[filter_value.size() - 2] == '.' &&
+ filter_value[filter_value.size() - 1] == '*';
+ if (is_prefix_match) {
+ filter_value.erase(filter_value.size() - 2);
+ }
+
std::vector<void*> raw_posting_lists;
// there could be multiple tokens in a filter value, which we have to treat as ANDs
@@ -1162,6 +1168,10 @@ void filter_result_iterator_t::init() {
}
str_tokens.push_back(str_token);
+ if (is_prefix_match) {
+ continue;
+ }
+
art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
str_token.length()+1);
if (leaf == nullptr) {
@@ -1177,7 +1187,84 @@ void filter_result_iterator_t::init() {
status = Option<bool>(400, "Error with filter field `" + f.name + "`: Filter value cannot be empty.");
validity = invalid;
return;
- } else if (raw_posting_lists.size() != str_tokens.size()) {
+ }
+
+ if (is_prefix_match) {
+ std::vector<search_field_t> fq_fields;
+ fq_fields.emplace_back(f.name, f.name, 1, 0, true, enable_t::off);
+
+ std::vector<token_t> value_tokens;
+ for (size_t i = 0; i < str_tokens.size(); i++) {
+ value_tokens.emplace_back(i, str_tokens[i], false, str_tokens[i].size(), 0);
+ }
+ value_tokens.back().is_prefix_searched = true;
+
+ filter_result_iterator_t filter_result_it(nullptr, 0);
+ std::vector<sort_by> sort_fields;
+ std::vector<std::vector<art_leaf*>> searched_filters;
+ tsl::htrie_map<char, token_leaf> qtoken_set;
+ Topster* topster = nullptr;
+ spp::sparse_hash_map<uint64_t, uint32_t> groups_processed;
+ uint32_t* all_result_ids = nullptr;
+ size_t all_result_ids_len = 0;
+ std::vector<std::string> group_by_fields;
+ std::set<uint64> query_hashes;
+ size_t typo_tokens_threshold = 0;
+ size_t max_candidates = 4;
+ size_t min_len_1typo = 0;
+ size_t min_len_2typo = 0;
+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};
+ const std::vector<size_t> geopoint_indices;
+
+ auto fuzzy_search_fields_op = index->fuzzy_search_fields(fq_fields, value_tokens, {}, text_match_type_t::max_score,
+ nullptr, 0, &filter_result_it, {}, {}, sort_fields,
+ {0}, searched_filters, qtoken_set, topster,
+ groups_processed, all_result_ids, all_result_ids_len,
+ 0, group_by_fields, false, false, false, false,
+ query_hashes, MAX_SCORE, {true}, typo_tokens_threshold,
+ false, max_candidates, min_len_1typo, min_len_2typo,
+ 0, nullptr, field_values, geopoint_indices, "", false);
+ delete[] all_result_ids;
+ if(!fuzzy_search_fields_op.ok()) {
+ continue;
+ }
+
+ // Searching for `Chris P.*` will return `Chris Parnell` and `Chris Pine`.
+ for (const auto& searched_filter_value: searched_filters) {
+ for (const auto& leaf: searched_filter_value) {
+ if (leaf == nullptr) {
+ continue;
+ }
+
+ // Tokens of a filter value get AND.
+ approx_filter_value_match = std::min(posting_t::num_ids(leaf->values), approx_filter_value_match);
+ raw_posting_lists.push_back(leaf->values);
+ }
+
+ if (raw_posting_lists.size() != str_tokens.size()) {
+ continue;
+ }
+
+ std::vector<posting_list_t*> plists;
+ posting_t::to_expanded_plists(raw_posting_lists, plists, expanded_plists);
+ if (plists.empty()) {
+ continue;
+ }
+
+ posting_lists.push_back(plists);
+ posting_list_iterators.emplace_back(std::vector<posting_list_t::iterator_t>());
+ for (auto const& plist: plists) {
+ posting_list_iterators.back().push_back(plist->new_iterator());
+ }
+
+ // Multiple filter values get OR.
+ approx_filter_ids_length += approx_filter_value_match;
+ raw_posting_lists.clear();
+ }
+ continue;
+ }
+
+ if (raw_posting_lists.size() != str_tokens.size()) {
continue;
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 13e1bf3d..e009d7be 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -2356,3 +2356,106 @@ TEST_F(CollectionFilteringTest, ComputeFilterResult) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionFilteringTest, PrefixFilterOnTextFields) {
+ Collection *coll_mul_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("cast", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)
+ };
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+
+ nlohmann::json results = coll_mul_fields->search("*", {}, "cast: Chris", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ std::vector<std::string> ids = {"6", "1", "7"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
+ results = coll_mul_fields->search("*", {}, "cast: Ch.*", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"6", "1", "7"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
+ results = coll_mul_fields->search("*", {}, "cast: M.*", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"3", "2", "16"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
+ results = coll_mul_fields->search("*", {}, "cast: Chris P.*", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ ids = {"1", "7"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
+ results = coll_mul_fields->search("*", {}, "cast: [Martin, Chris P.*]", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"2", "1", "7"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+
+ results = coll_mul_fields->search("*", {}, "cast: [M.*, Chris P.*]", {}, {}, {0},
+ 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(5, results["hits"].size());
+
+ ids = {"3", "2", "16", "1", "7"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_EQ(id, result_id);
+ }
+}