Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support prefix matching of filter value. (#1597)
Harpreet Sangardaad8df50beb
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 033ed005..5c904d4b 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1144,7 +1144,13 @@ void filter_result_iterator_t::init() {} else if (f.is_string()) {art_tree* t = index->search_index.at(a_filter.field_name);- for (const std::string& filter_value : a_filter.values) {+ for (std::string filter_value : a_filter.values) {+ auto is_prefix_match = filter_value.size() > 2 && filter_value[filter_value.size() - 2] == '.' &&+ filter_value[filter_value.size() - 1] == '*';+ if (is_prefix_match) {+ filter_value.erase(filter_value.size() - 2);+ }+std::vector<void*> raw_posting_lists;// there could be multiple tokens in a filter value, which we have to treat as ANDs@@ -1162,6 +1168,10 @@ void filter_result_iterator_t::init() {}str_tokens.push_back(str_token);+ if (is_prefix_match) {+ continue;+ }+art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),str_token.length()+1);if (leaf == nullptr) {@@ -1177,7 +1187,84 @@ void filter_result_iterator_t::init() {status = Option<bool>(400, "Error with filter field `" + f.name + "`: Filter value cannot be empty.");validity = invalid;return;- } else if (raw_posting_lists.size() != str_tokens.size()) {+ }++ if (is_prefix_match) {+ std::vector<search_field_t> fq_fields;+ fq_fields.emplace_back(f.name, f.name, 1, 0, true, enable_t::off);++ std::vector<token_t> value_tokens;+ for (size_t i = 0; i < str_tokens.size(); i++) {+ value_tokens.emplace_back(i, str_tokens[i], false, str_tokens[i].size(), 0);+ }+ value_tokens.back().is_prefix_searched = true;++ filter_result_iterator_t filter_result_it(nullptr, 0);+ std::vector<sort_by> sort_fields;+ std::vector<std::vector<art_leaf*>> searched_filters;+ tsl::htrie_map<char, token_leaf> qtoken_set;+ Topster* topster = nullptr;+ spp::sparse_hash_map<uint64_t, uint32_t> groups_processed;+ uint32_t* all_result_ids = nullptr;+ size_t all_result_ids_len = 0;+ std::vector<std::string> group_by_fields;+ std::set<uint64> query_hashes;+ size_t typo_tokens_threshold = 0;+ size_t max_candidates = 4;+ size_t min_len_1typo = 0;+ size_t min_len_2typo = 0;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values{};+ const std::vector<size_t> geopoint_indices;++ auto fuzzy_search_fields_op = index->fuzzy_search_fields(fq_fields, value_tokens, {}, text_match_type_t::max_score,+ nullptr, 0, &filter_result_it, {}, {}, sort_fields,+ {0}, searched_filters, qtoken_set, topster,+ groups_processed, all_result_ids, all_result_ids_len,+ 0, group_by_fields, false, false, false, false,+ query_hashes, MAX_SCORE, {true}, typo_tokens_threshold,+ false, max_candidates, min_len_1typo, min_len_2typo,+ 0, nullptr, field_values, geopoint_indices, "", false);+ delete[] all_result_ids;+ if(!fuzzy_search_fields_op.ok()) {+ continue;+ }++ // Searching for `Chris P.*` will return `Chris Parnell` and `Chris Pine`.+ for (const auto& searched_filter_value: searched_filters) {+ for (const auto& leaf: searched_filter_value) {+ if (leaf == nullptr) {+ continue;+ }++ // Tokens of a filter value get AND.+ approx_filter_value_match = std::min(posting_t::num_ids(leaf->values), approx_filter_value_match);+ raw_posting_lists.push_back(leaf->values);+ }++ if (raw_posting_lists.size() != str_tokens.size()) {+ continue;+ }++ std::vector<posting_list_t*> plists;+ posting_t::to_expanded_plists(raw_posting_lists, plists, expanded_plists);+ if (plists.empty()) {+ continue;+ }++ posting_lists.push_back(plists);+ posting_list_iterators.emplace_back(std::vector<posting_list_t::iterator_t>());+ for (auto const& plist: plists) {+ posting_list_iterators.back().push_back(plist->new_iterator());+ }++ // Multiple filter values get OR.+ approx_filter_ids_length += approx_filter_value_match;+ raw_posting_lists.clear();+ }+ continue;+ }++ if (raw_posting_lists.size() != str_tokens.size()) {continue;}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 13e1bf3d..e009d7be 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2356,3 +2356,106 @@ TEST_F(CollectionFilteringTest, ComputeFilterResult) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionFilteringTest, PrefixFilterOnTextFields) {+ Collection *coll_mul_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)+ };++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_mul_fields->add(json_line);+ }++ infile.close();++ nlohmann::json results = coll_mul_fields->search("*", {}, "cast: Chris", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(3, results["hits"].size());++ std::vector<std::string> ids = {"6", "1", "7"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ results = coll_mul_fields->search("*", {}, "cast: Ch.*", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(3, results["hits"].size());++ ids = {"6", "1", "7"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ results = coll_mul_fields->search("*", {}, "cast: M.*", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(3, results["hits"].size());++ ids = {"3", "2", "16"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ results = coll_mul_fields->search("*", {}, "cast: Chris P.*", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"1", "7"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ results = coll_mul_fields->search("*", {}, "cast: [Martin, Chris P.*]", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(3, results["hits"].size());++ ids = {"2", "1", "7"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ results = coll_mul_fields->search("*", {}, "cast: [M.*, Chris P.*]", {}, {}, {0},+ 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(5, results["hits"].size());++ ids = {"3", "2", "16", "1", "7"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }+}