Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Implement facet filtering.
kishorenc4fa89ebf007f
diff --git a/include/collection.h b/include/collection.hindex c97c9b36..1c478e09 100644--- a/include/collection.h+++ b/include/collection.h@@ -221,7 +221,8 @@ public:size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),- size_t max_facet_values=10, size_t max_hits=500);+ size_t max_facet_values=10, size_t max_hits=500,+ const std::string & simple_facet_query = "");Option<nlohmann::json> get(const std::string & id);diff --git a/include/field.h b/include/field.hindex a011b0fe..162151a8 100644--- a/include/field.h+++ b/include/field.h@@ -133,4 +133,9 @@ struct facet {facet(const std::string & field_name): field_name(field_name) {}+};++struct facet_query_t {+ std::string field_name;+ std::string query;};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex f77fdbc2..e91e35e4 100644--- a/include/index.h+++ b/include/index.h@@ -29,6 +29,7 @@ struct search_args {std::vector<uint32_t> included_ids;std::vector<uint32_t> excluded_ids;std::vector<sort_by> sort_fields_std;+ facet_query_t facet_query;int num_typos;size_t max_facet_values;size_t max_hits;@@ -49,11 +50,11 @@ struct search_args {search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,- std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,+ std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,size_t drop_tokens_threshold):query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),- excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), num_typos(num_typos),+ excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),max_facet_values(max_facet_values), max_hits(max_hits), per_page(per_page),page(page), token_order(token_order), prefix(prefix), drop_tokens_threshold(drop_tokens_threshold),all_result_ids_len(0), outcome(0) {@@ -139,7 +140,8 @@ private:Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);- void do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size);+ void do_facets(std::vector<facet> & facets, const facet_query_t & facet_query,+ const uint32_t* result_ids, size_t results_size);void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> & ids);@@ -202,6 +204,7 @@ public:void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,+ const facet_query_t & facet_query,const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,const std::vector<sort_by> & sort_fields_std, const int num_typos,const size_t max_hits, const size_t per_page, const size_t page, const token_ordering token_order,@@ -271,8 +274,9 @@ public:search_args search_params;- static void- populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions,- const art_leaf *token_leaf, uint32_t doc_index);+ static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,+ const art_leaf *token_leaf, uint32_t doc_index);++ int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;};diff --git a/src/collection.cpp b/src/collection.cppindex 27db4b63..62510d6e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -310,7 +310,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const size_t drop_tokens_threshold,const spp::sparse_hash_set<std::string> & include_fields,const spp::sparse_hash_set<std::string> & exclude_fields,- const size_t max_facet_values, const size_t max_hits) {+ const size_t max_facet_values, const size_t max_hits,+ const std::string & simple_facet_query) {std::vector<uint32_t> included_ids;std::vector<uint32_t> excluded_ids;@@ -453,6 +454,24 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::facets.emplace_back(field_name);}+ // parse facet query+ std::vector<std::string> facet_query_vec;+ facet_query_t facet_query;++ if(!simple_facet_query.empty() && simple_facet_query.find(':') == std::string::npos) {+ std::string error = "Facet query must be in the `facet_field: value` format.";+ return Option<nlohmann::json>(404, error);+ }++ StringUtils::split(simple_facet_query, facet_query_vec, ":");+ if(!facet_query_vec.empty()) {+ facet_query = { StringUtils::trim(facet_query_vec[0]), StringUtils::trim(facet_query_vec[1]) };+ if(facet_schema.count(facet_query.field_name) == 0) {+ std::string error = "Could not find a facet field named `" + facet_query.field_name + "` in the schema.";+ return Option<nlohmann::json>(404, error);+ }+ }+// validate sort fields and standardizestd::vector<sort_by> sort_fields_std;@@ -502,7 +521,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::for(Index* index: indices) {index->search_params = search_args(query, search_fields, filters, facets,index_to_included_ids[index_id], index_to_excluded_ids[index_id],- sort_fields_std, num_typos, max_facet_values, max_hits,+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,per_page, page, token_order, prefix, drop_tokens_threshold);{std::lock_guard<std::mutex> lk(index->m);diff --git a/src/core_api.cpp b/src/core_api.cppindex 07d73821..4ed6f085 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -174,6 +174,7 @@ void get_search(http_req & req, http_res & res) {const char *SORT_BY = "sort_by";const char *FACET_BY = "facet_by";+ const char *FACET_QUERY = "facet_query";const char *MAX_FACET_VALUES = "max_facet_values";const char *MAX_HITS = "max_hits";@@ -252,6 +253,10 @@ void get_search(http_req & req, http_res & res) {std::vector<std::string> facet_fields;StringUtils::split(req.params[FACET_BY], facet_fields, ",");+ if(req.params.count(FACET_QUERY) == 0) {+ req.params[FACET_QUERY] = "";+ }+std::vector<std::string> include_fields_vec;StringUtils::split(req.params[INCLUDE_FIELDS], include_fields_vec, ",");@@ -307,7 +312,8 @@ void get_search(http_req & req, http_res & res) {token_order, prefix, drop_tokens_threshold,include_fields, exclude_fields,static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),- static_cast<size_t>(std::stoi(req.params[MAX_HITS])));+ static_cast<size_t>(std::stoi(req.params[MAX_HITS])),+ req.params[FACET_QUERY]);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/index.cpp b/src/index.cppindex c33a4aa9..e2670820 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -522,7 +522,8 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin}}-void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size) {+void Index::do_facets(std::vector<facet> & facets, const facet_query_t & facet_query,+ const uint32_t* result_ids, size_t results_size) {std::map<std::string, size_t> facet_to_index;size_t i_facet = 0;@@ -533,12 +534,35 @@ void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, sfor(auto & a_facet: facets) {// assumed that facet fields have already been validated upstream+ spp::sparse_hash_set<int64_t> facet_filter_set;+ bool use_facet_query = false;++ if(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {+ use_facet_query = true;+ int bounded_cost = get_bounded_typo_cost(2, facet_query.query.size());+ std::vector<art_leaf*> leaves;+ art_fuzzy_search(search_index.at(a_facet.field_name), (const unsigned char *) facet_query.query.c_str(),+ facet_query.query.size(),0, bounded_cost, 10000,+ token_ordering::MAX_SCORE, true, leaves);++ for(const auto & leaf: leaves) {+ // calculate hash without terminating null char+ uint64_t hash = wyhash(leaf->key, leaf->key_len-1, 0, _wyp);+ facet_filter_set.insert(hash);+ }+ }+for(size_t i = 0; i < results_size; i++) {uint32_t doc_seq_id = result_ids[i];+if(facet_index_v2.count(doc_seq_id) != 0) {const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];for(size_t j = 0; j < values.size(); j++) {+ if(use_facet_query && facet_filter_set.find(values[j]) == facet_filter_set.end()) {+ // this particular facet value is not found in facet filter, so ignore+ continue;+ }a_facet.result_map[values[j]].count += 1;a_facet.result_map[values[j]].doc_id = doc_seq_id;a_facet.result_map[values[j]].array_pos = j;@@ -804,7 +828,7 @@ void Index::run_search() {// after the wait, we own the lock.search(search_params.outcome, search_params.query, search_params.search_fields,- search_params.filters, search_params.facets, search_params.included_ids,+ search_params.filters, search_params.facets, search_params.facet_query, search_params.included_ids,search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,search_params.max_hits, search_params.per_page, search_params.page, search_params.token_order,search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,@@ -887,7 +911,8 @@ void Index::collate_curated_ids(const std::string & query, const std::string & fvoid Index::search(Option<uint32_t> & outcome,std::string query,const std::vector<std::string> & search_fields,- const std::vector<filter> & filters, std::vector<facet> & facets,+ const std::vector<filter> & filters,+ std::vector<facet> & facets, const facet_query_t & facet_query,const std::vector<uint32_t> & included_ids,const std::vector<uint32_t> & excluded_ids,const std::vector<sort_by> & sort_fields_std, const int num_typos, const size_t max_hits,@@ -925,13 +950,13 @@ void Index::search(Option<uint32_t> & outcome,score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},filter_ids, filter_ids_length);collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);- do_facets(facets, filter_ids, filter_ids_length);+ do_facets(facets, facet_query, filter_ids, filter_ids_length);all_result_ids_len = filter_ids_length;} else {const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);for(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces results- if(filters.size() == 0 || filter_ids_length > 0) {+ if(filters.empty() || filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort resultsconst std::string & field = search_fields[i];@@ -941,10 +966,10 @@ void Index::search(Option<uint32_t> & outcome,collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);}}- do_facets(facets, all_result_ids, all_result_ids_len);+ do_facets(facets, facet_query, all_result_ids, all_result_ids_len);}- do_facets(facets, &included_ids[0], included_ids.size());+ do_facets(facets, facet_query, &included_ids[0], included_ids.size());// must be sorted before iterated upon to remove "empty" array entriestopster.sort();@@ -1019,10 +1044,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const stconst size_t token_len = tokens[token_index].length();// This ensures that we don't end up doing a cost of 1 for a single char etc.- int bounded_cost = max_cost;- if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {- bounded_cost = token_len - 1;- }+ int bounded_cost = get_bounded_typo_cost(max_cost, token_len);for(int cost = 0; cost <= bounded_cost; cost++) {all_costs.push_back(cost);@@ -1154,6 +1176,14 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st}}+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const {+ int bounded_cost = max_cost;+ if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {+ bounded_cost = token_len - 1;+ }+ return bounded_cost;+}+void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {LOG(INFO) << "Token: " << token << ", cost: " << cost;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 955c5b95..146aaed6 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1843,6 +1843,47 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ // facet with facet filter query (allows typo correction!)+ results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, " tags : sliver").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ // facet query that does not match any indexed value+ results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, " tags : notfound").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(0, results["facet_counts"][0]["counts"].size());++ // bad facet query syntax+ auto res_op = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "foobar");++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Facet query must be in the `facet_field: value` format.", res_op.error().c_str());++ // unknown facet field+ res_op = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "foobar: baz");++ ASSERT_FALSE(res_op.ok());+ ASSERT_STREQ("Could not find a facet field named `foobar` in the schema.", res_op.error().c_str());+collectionManager.drop_collection("coll_array_fields");}